From e4eba157c332b445bff82726f66deecb495942b2 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Mon, 25 Jul 2022 09:36:45 +0800
Subject: [PATCH 001/232] feat: add emqx enterprise app connector & bridge

---
 lib-ee/emqx_ee_bridge/.gitignore              | 19 +++++++++++++
 lib-ee/emqx_ee_bridge/README.md               |  9 +++++++
 lib-ee/emqx_ee_bridge/rebar.config            |  6 +++++
 .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 14 ++++++++++
 .../emqx_ee_bridge/src/emqx_ee_bridge_app.erl | 17 ++++++++++++
 .../emqx_ee_bridge/src/emqx_ee_bridge_sup.erl | 27 +++++++++++++++++++
 lib-ee/emqx_ee_connector/.gitignore           | 19 +++++++++++++
 lib-ee/emqx_ee_connector/README.md            |  9 +++++++
 lib-ee/emqx_ee_connector/rebar.config         |  6 +++++
 .../src/emqx_ee_connector.app.src             | 14 ++++++++++
 .../src/emqx_ee_connector_app.erl             | 17 ++++++++++++
 .../src/emqx_ee_connector_sup.erl             | 27 +++++++++++++++++++
 .../src/emqx_enterprise_conf.app.src          |  1 -
 rebar.config.erl                              |  4 ++-
 14 files changed, 187 insertions(+), 2 deletions(-)
 create mode 100644 lib-ee/emqx_ee_bridge/.gitignore
 create mode 100644 lib-ee/emqx_ee_bridge/README.md
 create mode 100644 lib-ee/emqx_ee_bridge/rebar.config
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl
 create mode 100644 lib-ee/emqx_ee_connector/.gitignore
 create mode 100644 lib-ee/emqx_ee_connector/README.md
 create mode 100644 lib-ee/emqx_ee_connector/rebar.config
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl

diff --git a/lib-ee/emqx_ee_bridge/.gitignore b/lib-ee/emqx_ee_bridge/.gitignore
new file mode 100644
index 000000000..f1c455451
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/.gitignore
@@ -0,0 +1,19 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~
diff --git a/lib-ee/emqx_ee_bridge/README.md b/lib-ee/emqx_ee_bridge/README.md
new file mode 100644
index 000000000..5cb4d8694
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/README.md
@@ -0,0 +1,9 @@
+emqx_ee_bridge
+=====
+
+An OTP application
+
+Build
+-----
+
+    $ rebar3 compile
diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config
new file mode 100644
index 000000000..5dd22ccef
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@@ -0,0 +1,6 @@
+{erl_opts, [debug_info]}.
+{deps, []}.
+
+{shell, [
+    {apps, [emqx_ee_bridge]}
+]}.
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
new file mode 100644
index 000000000..58846577e
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -0,0 +1,14 @@
+{application, emqx_ee_bridge, [
+    {description, "An OTP application"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {mod, {emqx_ee_bridge_app, []}},
+    {applications, [
+        kernel,
+        stdlib
+    ]},
+    {env, []},
+    {modules, []},
+
+    {links, []}
+]}.
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl
new file mode 100644
index 000000000..22268e285
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl
@@ -0,0 +1,17 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_bridge_app).
+
+-behaviour(application).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    emqx_ee_bridge_sup:start_link().
+
+stop(_State) ->
+    ok.
+
+%% internal functions
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl
new file mode 100644
index 000000000..5a2484442
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl
@@ -0,0 +1,27 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_bridge_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+
+-export([init/1]).
+
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_all,
+        intensity => 0,
+        period => 1
+    },
+    ChildSpecs = [],
+    {ok, {SupFlags, ChildSpecs}}.
+
+%% internal functions
diff --git a/lib-ee/emqx_ee_connector/.gitignore b/lib-ee/emqx_ee_connector/.gitignore
new file mode 100644
index 000000000..f1c455451
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/.gitignore
@@ -0,0 +1,19 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~
diff --git a/lib-ee/emqx_ee_connector/README.md b/lib-ee/emqx_ee_connector/README.md
new file mode 100644
index 000000000..e665af458
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/README.md
@@ -0,0 +1,9 @@
+emqx_ee_connector
+=====
+
+An OTP application
+
+Build
+-----
+
+    $ rebar3 compile
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
new file mode 100644
index 000000000..cba9da867
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -0,0 +1,6 @@
+{erl_opts, [debug_info]}.
+{deps, []}.
+
+{shell, [
+    {apps, [emqx_ee_connector]}
+]}.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
new file mode 100644
index 000000000..87f68afb6
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -0,0 +1,14 @@
+{application, emqx_ee_connector, [
+    {description, "An OTP application"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {mod, {emqx_ee_connector_app, []}},
+    {applications, [
+        kernel,
+        stdlib
+    ]},
+    {env, []},
+    {modules, []},
+
+    {links, []}
+]}.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl
new file mode 100644
index 000000000..13db2ccb0
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl
@@ -0,0 +1,17 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_connector_app).
+
+-behaviour(application).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    emqx_ee_connector_sup:start_link().
+
+stop(_State) ->
+    ok.
+
+%% internal functions
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl
new file mode 100644
index 000000000..f8a219fdc
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl
@@ -0,0 +1,27 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_connector_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+
+-export([init/1]).
+
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_all,
+        intensity => 0,
+        period => 1
+    },
+    ChildSpecs = [],
+    {ok, {SupFlags, ChildSpecs}}.
+
+%% internal functions
diff --git a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src b/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src
index 37cb78b54..a25ce1aae 100644
--- a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src
+++ b/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src
@@ -9,6 +9,5 @@
     {env, []},
     {modules, []},
 
-    {licenses, ["Apache 2.0"]},
     {links, []}
 ]}.
diff --git a/rebar.config.erl b/rebar.config.erl
index 6748f7ce8..96754f7f8 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -378,7 +378,9 @@ is_app(Name) ->
 relx_apps_per_edition(ee) ->
     [
         emqx_license,
-        {emqx_enterprise_conf, load}
+        {emqx_enterprise_conf, load},
+        emqx_ee_connector,
+        emqx_ee_bridge
     ];
 relx_apps_per_edition(ce) ->
     [].

From f00a7417bb6eb7c0754cfa43e29919d938e29014 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Mon, 25 Jul 2022 11:18:38 +0800
Subject: [PATCH 002/232] feat: hstream bridge & connector

---
 apps/emqx_bridge/src/emqx_bridge_schema.erl   |  14 +-
 .../i18n/emqx_ee_bridge_hstream.conf          |  84 +++++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  15 ++
 .../src/emqx_ee_bridge_hstream.erl            |  57 +++++
 .../i18n/emqx_ee_connector_hstream.conf       |  43 ++++
 lib-ee/emqx_ee_connector/rebar.config         |   4 +-
 .../src/emqx_ee_connector_hstream.erl         | 218 ++++++++++++++++++
 7 files changed, 430 insertions(+), 5 deletions(-)
 create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
 create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl

diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl
index d512df323..d162f0c13 100644
--- a/apps/emqx_bridge/src/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl
@@ -45,10 +45,16 @@ http_schema(Method) ->
         end,
         ?CONN_TYPES
     ),
-    hoconsc:union([
-        ref(emqx_bridge_webhook_schema, Method)
-        | Schemas
-    ]).
+    ExtSchemas = [ref(Module, Method) || Module <- schema_modules()],
+    hoconsc:union(Schemas ++ ExtSchemas).
+
+-ifdef(EMQX_RELEASE_EDITION).
+schema_modules() ->
+    [emqx_bridge_webhook_schema] ++ emqx_ee_bridge:schema_modules().
+-else.
+schema_modules() ->
+    [emqx_bridge_webhook_schema].
+-endif.
 
 common_bridge_fields(ConnectorRef) ->
     [
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
new file mode 100644
index 000000000..a110dbf27
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
@@ -0,0 +1,84 @@
+emqx_ee_bridge_hstream {
+    local_topic {
+        desc {
+            en: """
+The MQTT topic filter to be forwarded to the HStreamDB. All MQTT 'PUBLISH' messages with the topic
+matching the local_topic will be forwarded.</br>
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is
+configured, then both the data got from the rule and the MQTT messages that match local_topic
+will be forwarded.
+"""
+            zh: """
+发送到 'local_topic' 的消息都会转发到 HStreamDB。 </br>
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。
+"""
+        }
+    label {
+            en: "Local Topic"
+            zh: "本地 Topic"
+        }
+    }
+    payload {
+        desc {
+            en: """The payload to be forwarded to the HStreamDB. Placeholders supported."""
+            zh: """要转发到 HStreamDB 的数据内容，支持占位符"""
+            }
+        label {
+            en: "Payload"
+            zh: "消息内容"
+        }
+    }
+    config_enable {
+        desc {
+            en: """Enable or disable this bridge"""
+            zh: """启用/禁用 Bridge"""
+        }
+        label {
+            en: "Enable Or Disable Bridge"
+            zh: "启用/禁用 Bridge"
+        }
+        }
+    config_direction {
+        desc {
+            en: """The direction of this bridge, MUST be 'egress'"""
+            zh: """Bridge 的方向， 必须是 egress"""
+        }
+        label {
+            en: "Bridge Direction"
+            zh: "Bridge 方向"
+        }
+    }
+
+    desc_config {
+        desc {
+            en: """Configuration for an HStreamDB bridge."""
+            zh: """HStreamDB Bridge 配置"""
+        }
+        label: {
+            en: "HStreamDB Bridge Configuration"
+            zh: "HStreamDB Bridge 配置"
+        }
+    }
+
+    desc_type {
+        desc {
+            en: """The Bridge Type"""
+            zh: """Bridge 类型"""
+        }
+        label {
+            en: "Bridge Type"
+            zh: "Bridge 类型"
+        }
+    }
+
+    desc_name {
+        desc {
+            en: """Bridge name, used as a human-readable description of the bridge."""
+            zh: """Bridge 名字，Bridge 的可读描述"""
+        }
+        label {
+            en: "Bridge Name"
+            zh: "Bridge 名字"
+        }
+    }
+}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
new file mode 100644
index 000000000..d18fa1fde
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -0,0 +1,15 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge).
+
+-export([
+    schema_modules/0,
+    info_example_basic/2
+]).
+
+schema_modules() ->
+    [emqx_ee_bridge_hstream].
+
+info_example_basic(_Type, _Direction) ->
+    #{}.
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
new file mode 100644
index 000000000..6b01d2c0b
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
@@ -0,0 +1,57 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_hstream).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+
+-import(hoconsc, [mk/2, enum/1]).
+
+-export([
+    roots/0,
+    fields/1,
+    desc/1
+]).
+
+%%======================================================================================
+%% Hocon Schema Definitions
+% namespace() -> "bridge".
+
+roots() -> [].
+
+fields("config") ->
+    ExtConfig = [
+        {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
+        {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})}
+    ],
+    basic_config() ++ ExtConfig;
+fields("post") ->
+    [type_field(), name_field() | fields("config")];
+fields("put") ->
+    fields("config");
+fields("get") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields("post").
+
+desc("config") ->
+    ?DESC("desc_config");
+desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
+    ["Configuration for HStream using `", string:to_upper(Method), "` method."];
+desc(_) ->
+    undefined.
+
+basic_config() ->
+    Basic = [
+        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}
+    ],
+    emqx_ee_connector_hstream:fields(config) ++ Basic.
+
+%%======================================================================================
+
+type_field() ->
+    % {type, mk(hstream, #{required => true, desc => ?DESC("desc_type")})}.
+    {type, mk(hstream, #{required => true, desc => <<"desc_type">>})}.
+
+name_field() ->
+    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
new file mode 100644
index 000000000..d99b09c5f
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
@@ -0,0 +1,43 @@
+
+emqx_ee_connector_hstream {
+    url {
+        desc {
+                en: """Hstream Server URL"""
+                zh: """HStream 服务器 URL"""
+            }
+        label {
+                en: """Hstream Server URL"""
+                zh: """HStream 服务器 URL"""
+                }
+        }
+    stream_name {
+        desc {
+                en: """Hstream Stream Name"""
+                zh: """HStream 流名称"""
+            }
+        label {
+                en: """Hstream Stream Name"""
+                zh: """HStream 流名称"""
+                }
+    }
+    ordering_key {
+        desc {
+                en: """Hstream Ordering Key"""
+                zh: """HStream 分区键"""
+            }
+        label {
+                en: """Hstream Ordering Key"""
+                zh: """HStream 分区键"""
+                }
+    }
+    pool_size {
+        desc {
+                en: """Hstream Pool Size"""
+                zh: """HStream 连接池大小"""
+            }
+        label {
+                en: """Hstream Pool Size"""
+                zh: """HStream 连接池大小"""
+                }
+    }
+}
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index cba9da867..1f16e648e 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -1,5 +1,7 @@
 {erl_opts, [debug_info]}.
-{deps, []}.
+{deps, [
+  {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.4"}}}
+]}.
 
 {shell, [
     {apps, [emqx_ee_connector]}
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
new file mode 100644
index 000000000..ea3a73035
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
@@ -0,0 +1,218 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_connector_hstream).
+
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+-import(hoconsc, [mk/2, enum/1]).
+
+-behaviour(emqx_resource).
+
+%% callbacks of behaviour emqx_resource
+-export([
+    on_start/2,
+    on_stop/2,
+    on_query/4,
+    on_get_status/2
+]).
+
+-export([
+    on_flush_result/1
+]).
+
+-export([
+    roots/0,
+    fields/1
+]).
+
+%% -------------------------------------------------------------------------------------------------
+%% resource callback
+
+on_start(InstId, Config) ->
+    start_client(InstId, Config).
+
+on_stop(InstId, #{client := Client, producer := Producer}) ->
+    StopClientRes = hstreamdb:stop_client(Client),
+    StopProducerRes = hstreamdb:stop_producer(Producer),
+    ?SLOG(info, #{
+        msg => "stop hstream connector",
+        connector => InstId,
+        client => Client,
+        producer => Producer,
+        stop_client => StopClientRes,
+        stop_producer => StopProducerRes
+    }).
+
+on_query(_InstId, {OrderingKey, Payload, Record}, AfterQuery, #{producer := Producer}) ->
+    Record = hstreamdb:to_record(OrderingKey, raw, Payload),
+    do_append(AfterQuery, false, Producer, Record).
+
+on_get_status(_InstId, #{client := Client}) ->
+    is_alive(Client).
+
+%% -------------------------------------------------------------------------------------------------
+%% hstream batch callback
+%% TODO: maybe remove it after disk cache is ready
+
+on_flush_result({{flush, _Stream, _Records}, {ok, _Resp}}) ->
+    ok;
+on_flush_result({{flush, _Stream, _Records}, {error, _Reason}}) ->
+    ok.
+
+%% -------------------------------------------------------------------------------------------------
+%% schema
+
+roots() ->
+    fields(config).
+
+fields(config) ->
+    [
+        {url, mk(binary(), #{required => true, desc => ?DESC("url")})},
+        {stream, mk(binary(), #{required => true, desc => ?DESC("stream")})},
+        {ordering_key, mk(binary(), #{required => true, desc => ?DESC("ordering_key")})},
+        {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}
+    ].
+
+%% -------------------------------------------------------------------------------------------------
+%% internal functions
+
+start_client(InstId, Config = #{server := Server, pool_size := PoolSize}) ->
+    ?SLOG(info, #{
+        msg => "starting hstream connector: client",
+        connector => InstId,
+        config => Config
+    }),
+    ClientName = client_name(InstId),
+    ClientOptions = [
+        {url, Server},
+        {rpc_options, #{pool_size => PoolSize}}
+    ],
+    case hstreamdb:start_client(ClientName, ClientOptions) of
+        {ok, Client} ->
+            case is_alive(Client) of
+                true ->
+                    ?SLOG(info, #{
+                        msg => "hstream connector: client started",
+                        connector => InstId,
+                        client => Client
+                    }),
+                    start_producer(InstId, Client, Config);
+                _ ->
+                    ?SLOG(error, #{
+                        msg => "hstream connector: client not alive",
+                        connector => InstId
+                    }),
+                    {error, connect_failed}
+            end;
+        {error, {already_started, Pid}} ->
+            ?SLOG(info, #{
+                msg => "starting hstream connector: client, find old client. restart client",
+                old_client_pid => Pid,
+                old_client_name => ClientName
+            }),
+            _ = hstreamdb:stop_client(ClientName),
+            start_client(InstId, Config);
+        {error, Error} ->
+            ?SLOG(error, #{
+                msg => "hstream connector: client failed",
+                connector => InstId,
+                reason => Error
+            }),
+            {error, Error}
+    end.
+
+is_alive(Client) ->
+    case hstreamdb:echo(Client) of
+        {ok, _Echo} ->
+            true;
+        _ErrorEcho ->
+            false
+    end.
+
+start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSize}) ->
+    %% TODO: change these batch options after we have better disk cache.
+    BatchSize = maps:get(batch_size, Options, 100),
+    Interval = maps:get(batch_interval, Options, 1000),
+    ProducerOptions = [
+        {stream, Stream},
+        {callback, {?MODULE, on_flush_result, []}},
+        {max_records, BatchSize},
+        {interval, Interval},
+        {pool_size, PoolSize}
+    ],
+    Name = produce_name(InstId),
+    ?SLOG(info, #{
+        msg => "starting hstream connector: producer",
+        connector => InstId
+    }),
+    case hstreamdb:start_producer(Client, Name, ProducerOptions) of
+        {ok, Producer} ->
+            ?SLOG(info, #{
+                msg => "hstream connector: producer started"
+            }),
+            EnableBatch = maps:get(enable_batch, Options, false),
+            #{client => Client, producer => Producer, enable_batch => EnableBatch};
+        {error, {already_started, Pid}} ->
+            ?SLOG(info, #{
+                msg => "starting hstream connector: producer, find old producer. restart producer",
+                old_producer_pid => Pid,
+                old_producer_name => Name
+            }),
+            _ = hstreamdb:stop_producer(Name),
+            start_producer(InstId, Client, Options);
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "starting hstream connector: producer, failed",
+                reason => Reason
+            }),
+            {error, Reason}
+    end.
+
+%% TODO: this append is async, remove or change it after we have better disk cache.
+% do_append(AfterQuery, true, Producer, Record) ->
+%     case hstreamdb:append(Producer, Record) of
+%         ok ->
+%             ?SLOG(debug, #{
+%                 msg => "hstream producer async append success",
+%                 record => Record
+%             }),
+%             emqx_resource:query_success(AfterQuery);
+%         {error, Reason} ->
+%             ?SLOG(error, #{
+%                 msg => "hstream producer async append failed",
+%                 reason => Reason,
+%                 record => Record
+%             }),
+%             emqx_resource:query_failed(AfterQuery)
+%     end;
+do_append(AfterQuery, false, Producer, Record) ->
+    %% TODO: this append is sync, but it does not support [Record], can only append one Record.
+    %% Change it after we have better dick cache.
+    case hstreamdb:append_flush(Producer, Record) of
+        {ok, _} ->
+            ?SLOG(debug, #{
+                msg => "hstream producer sync append success",
+                record => Record
+            }),
+            emqx_resource:query_success(AfterQuery);
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "hstream producer sync append failed",
+                reason => Reason,
+                record => Record
+            }),
+            emqx_resource:query_failed(AfterQuery)
+    end.
+
+client_name(InstId) ->
+    "backend_hstream_client:" ++ to_string(InstId).
+
+produce_name(ActionId) ->
+    list_to_atom("backend_hstream_producer:" ++ to_string(ActionId)).
+
+to_string(List) when is_list(List) -> List;
+to_string(Bin) when is_binary(Bin) -> binary_to_list(Bin);
+to_string(Atom) when is_atom(Atom) -> atom_to_list(Atom).

From 98b36c468167135300e069989b1553db8fc40331 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Mon, 25 Jul 2022 19:03:53 +0800
Subject: [PATCH 003/232] fix: hstream db connector , TODO: start apps

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 16 +++++--
 apps/emqx_bridge/src/emqx_bridge_resource.erl | 24 ++++++++--
 apps/emqx_bridge/src/emqx_bridge_schema.erl   | 10 +++-
 .../src/emqx_resource_manager.erl             |  2 +-
 .../emqx_ee_bridge/include/emqx_ee_bridge.hrl | 18 ++++++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  | 31 +++++++++++--
 .../src/emqx_ee_bridge_hstream.erl            | 46 ++++++++++++++++---
 .../i18n/emqx_ee_connector_hstream.conf       |  2 +-
 .../src/emqx_ee_connector.app.src             |  3 +-
 .../src/emqx_ee_connector_hstream.erl         | 23 ++++++++--
 scripts/merge-i18n.escript                    |  2 +-
 11 files changed, 151 insertions(+), 26 deletions(-)
 create mode 100644 lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index bc9b6c5a2..0d3facc5c 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -163,7 +163,7 @@ bridge_info_examples(Method) ->
     }).
 
 conn_bridge_examples(Method) ->
-    lists:foldl(
+    Fun =
         fun(Type, Acc) ->
             SType = atom_to_list(Type),
             KeyIngress = bin(SType ++ "_ingress"),
@@ -179,9 +179,17 @@ conn_bridge_examples(Method) ->
                 }
             })
         end,
-        #{},
-        ?CONN_TYPES
-    ).
+    Broker = lists:foldl(Fun, #{}, ?CONN_TYPES),
+    EE = ee_conn_bridge_examples(Method),
+    maps:merge(Broker, EE).
+
+-ifdef(EMQX_RELEASE_EDITION).
+ee_conn_bridge_examples(Method) ->
+    emqx_ee_bridge:conn_bridge_examples(Method).
+-else.
+ee_conn_bridge_examples(_Method) ->
+    #{}.
+-endif.
 
 info_example(Type, Direction, Method) ->
     maps:merge(
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index 678aa1f10..c82a290b3 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -42,10 +42,18 @@
     reset_metrics/1
 ]).
 
+-ifdef(EMQX_RELEASE_EDITION).
+bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt;
+bridge_to_resource_type(mqtt) -> emqx_connector_mqtt;
+bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http;
+bridge_to_resource_type(webhook) -> emqx_connector_http;
+bridge_to_resource_type(BridgeType) -> emqx_ee_bridge:resource_type(BridgeType).
+-else.
 bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt;
 bridge_to_resource_type(mqtt) -> emqx_connector_mqtt;
 bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http;
 bridge_to_resource_type(webhook) -> emqx_connector_http.
+-endif.
 
 resource_id(BridgeId) when is_binary(BridgeId) ->
     <<"bridge:", BridgeId/binary>>.
@@ -254,7 +262,7 @@ parse_confs(
                 request_timeout => ReqTimeout
             }
     };
-parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) when
+parse_confs(Type = mqtt, Name, #{connector := ConnId, direction := Direction} = Conf) when
     is_binary(ConnId)
 ->
     case emqx_connector:parse_connector_id(ConnId) of
@@ -270,7 +278,7 @@ parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) w
         {_ConnType, _ConnName} ->
             error({cannot_use_connector_with_different_type, ConnId})
     end;
-parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when
+parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when
     is_map(ConnectorConfs)
 ->
     make_resource_confs(
@@ -279,7 +287,17 @@ parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} =
         maps:without([connector, direction], Conf),
         Type,
         Name
-    ).
+    );
+parse_confs(Type, Name, Conf) ->
+    parse_enterprise_confs(Type, Name, Conf).
+
+-ifdef(EMQX_RELEASE_EDITION).
+parse_enterprise_confs(Type, Name, Conf) ->
+    emqx_ee_bridge:parse_conf(Type, Name, Conf).
+-else.
+parse_enterprise_confs(Type, Name, Conf) ->
+    error({not_supported, Type, Name}).
+-endif.
 
 make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
     BName = bridge_id(Type, Name),
diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl
index d162f0c13..de27cd34c 100644
--- a/apps/emqx_bridge/src/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl
@@ -133,7 +133,7 @@ fields(bridges) ->
                     #{desc => ?DESC("bridges_name")}
                 )}
          || T <- ?CONN_TYPES
-        ];
+        ] ++ ee_fields_bridges();
 fields("metrics") ->
     [
         {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})},
@@ -158,6 +158,14 @@ fields("node_status") ->
         {"status", mk(status(), #{})}
     ].
 
+-ifdef(EMQX_RELEASE_EDITION).
+ee_fields_bridges() ->
+    emqx_ee_bridge:fields(bridges).
+-else.
+ee_fields_bridges() ->
+    [].
+-endif.
+
 desc(bridges) ->
     ?DESC("desc_bridges");
 desc("metrics") ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 3a1afd27c..82be9c58f 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -214,7 +214,7 @@ get_metrics(ResId) ->
 reset_metrics(ResId) ->
     emqx_metrics_worker:reset_metrics(resource_metrics, ResId).
 
-%% @doc Returns the data for all resorces
+%% @doc Returns the data for all resources
 -spec list_all() -> [resource_data()] | [].
 list_all() ->
     try
diff --git a/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl b/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl
new file mode 100644
index 000000000..0065db56b
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl
@@ -0,0 +1,18 @@
+-define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{
+    matched => MATCH,
+    success => SUCC,
+    failed => FAILED,
+    rate => RATE,
+    rate_last5m => RATE_5,
+    rate_max => RATE_MAX
+}).
+
+-define(METRICS_EXAMPLE, #{
+    metrics => ?METRICS(0, 0, 0, 0, 0, 0),
+    node_metrics => [
+        #{
+            node => node(),
+            metrics => ?METRICS(0, 0, 0, 0, 0, 0)
+        }
+    ]
+}).
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index d18fa1fde..a94089c21 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -3,13 +3,38 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_bridge).
 
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
 -export([
     schema_modules/0,
-    info_example_basic/2
+    conn_bridge_examples/1,
+    resource_type/1,
+    parse_conf/3,
+    fields/1
 ]).
 
 schema_modules() ->
     [emqx_ee_bridge_hstream].
 
-info_example_basic(_Type, _Direction) ->
-    #{}.
+conn_bridge_examples(Method) ->
+    Fun =
+        fun(Module, Examples) ->
+            Example = erlang:apply(Module, conn_bridge_example, [Method]),
+            maps:merge(Examples, Example)
+        end,
+    lists:foldl(Fun, #{}, schema_modules()).
+
+resource_type(hstream) -> emqx_ee_connector_hstream;
+resource_type(<<"hstream">>) -> emqx_ee_connector_hstream.
+
+parse_conf(_Type, _Name, Conf) ->
+    Conf.
+
+fields(bridges) ->
+    [
+        {hstream,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
+                #{desc => <<"hstream_webhook">>}
+            )}
+    ].
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
index 6b01d2c0b..42dd86f14 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
@@ -5,18 +5,53 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
+-include("emqx_ee_bridge.hrl").
 
 -import(hoconsc, [mk/2, enum/1]).
 
 -export([
+    conn_bridge_example/1
+]).
+
+-export([
+    namespace/0,
     roots/0,
     fields/1,
     desc/1
 ]).
 
-%%======================================================================================
+%% -------------------------------------------------------------------------------------------------
+%% api
+
+conn_bridge_example(Method) ->
+    #{
+        <<"hstream">> => #{
+            summary => <<"HStreamDB Bridge">>,
+            value => values(Method)
+        }
+    }.
+
+values(get) ->
+    maps:merge(values(post), ?METRICS_EXAMPLE);
+values(post) ->
+    #{
+        type => hstream,
+        name => <<"hstream_bridge_demo">>,
+        url => <<"http://127.0.0.1:6570">>,
+        stream => <<"stream1">>,
+        ordering_key => <<"${topic}">>,
+        pool_size => 8,
+        enable => true,
+        direction => egress,
+        local_topic => <<"local/topic/#">>,
+        payload => <<"${payload}">>
+    };
+values(put) ->
+    values(post).
+
+%% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
-% namespace() -> "bridge".
+namespace() -> "bridge".
 
 roots() -> [].
 
@@ -47,11 +82,10 @@ basic_config() ->
     ],
     emqx_ee_connector_hstream:fields(config) ++ Basic.
 
-%%======================================================================================
-
+%% -------------------------------------------------------------------------------------------------
+%% internal
 type_field() ->
-    % {type, mk(hstream, #{required => true, desc => ?DESC("desc_type")})}.
-    {type, mk(hstream, #{required => true, desc => <<"desc_type">>})}.
+    {type, mk(enum([hstream]), #{required => true, desc => ?DESC("desc_type")})}.
 
 name_field() ->
     {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
index d99b09c5f..605944997 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
@@ -38,6 +38,6 @@ emqx_ee_connector_hstream {
         label {
                 en: """Hstream Pool Size"""
                 zh: """HStream 连接池大小"""
-                }
+            }
     }
 }
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 87f68afb6..214488f80 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -5,7 +5,8 @@
     {mod, {emqx_ee_connector_app, []}},
     {applications, [
         kernel,
-        stdlib
+        stdlib,
+        hstreamdb_erl
     ]},
     {env, []},
     {modules, []},
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
index ea3a73035..7ba92f809 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
@@ -51,7 +51,12 @@ on_query(_InstId, {OrderingKey, Payload, Record}, AfterQuery, #{producer := Prod
     do_append(AfterQuery, false, Producer, Record).
 
 on_get_status(_InstId, #{client := Client}) ->
-    is_alive(Client).
+    case is_alive(Client) of
+        true ->
+            connected;
+        false ->
+            disconnected
+    end.
 
 %% -------------------------------------------------------------------------------------------------
 %% hstream batch callback
@@ -71,15 +76,23 @@ roots() ->
 fields(config) ->
     [
         {url, mk(binary(), #{required => true, desc => ?DESC("url")})},
-        {stream, mk(binary(), #{required => true, desc => ?DESC("stream")})},
+        {stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})},
         {ordering_key, mk(binary(), #{required => true, desc => ?DESC("ordering_key")})},
         {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}
     ].
 
 %% -------------------------------------------------------------------------------------------------
 %% internal functions
+start_client(InstId, Config) ->
+    try
+        do_start_client(InstId, Config)
+    catch
+        E:R:S ->
+            io:format("E:R:S ~p:~p ~n~p~n", [E, R, S]),
+            error(E)
+    end.
 
-start_client(InstId, Config = #{server := Server, pool_size := PoolSize}) ->
+do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) ->
     ?SLOG(info, #{
         msg => "starting hstream connector: client",
         connector => InstId,
@@ -87,7 +100,7 @@ start_client(InstId, Config = #{server := Server, pool_size := PoolSize}) ->
     }),
     ClientName = client_name(InstId),
     ClientOptions = [
-        {url, Server},
+        {url, binary_to_list(Server)},
         {rpc_options, #{pool_size => PoolSize}}
     ],
     case hstreamdb:start_client(ClientName, ClientOptions) of
@@ -154,7 +167,7 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi
                 msg => "hstream connector: producer started"
             }),
             EnableBatch = maps:get(enable_batch, Options, false),
-            #{client => Client, producer => Producer, enable_batch => EnableBatch};
+            {ok, #{client => Client, producer => Producer, enable_batch => EnableBatch}};
         {error, {already_started, Pid}} ->
             ?SLOG(info, #{
                 msg => "starting hstream connector: producer, find old producer. restart producer",
diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript
index 9f8ac91ff..493798738 100755
--- a/scripts/merge-i18n.escript
+++ b/scripts/merge-i18n.escript
@@ -4,7 +4,7 @@
 
 main(_) ->
     BaseConf = <<"">>,
-    Cfgs = get_all_cfgs("apps/"),
+    Cfgs = get_all_cfgs("apps/") ++ get_all_cfgs("lib-ee/"),
     Conf = [merge(BaseConf, Cfgs),
             io_lib:nl()
             ],

From fa4bc921ac0876622183258c28e2b868f775d8be Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Tue, 26 Jul 2022 15:28:56 +0800
Subject: [PATCH 004/232] fix: hstream db connector & bridge, TODO: SUITE

---
 apps/emqx_bridge/src/emqx_bridge_app.erl      | 11 ++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  2 +-
 .../src/emqx_ee_bridge_hstream.erl            |  4 +-
 .../src/emqx_ee_connector_hstream.erl         | 50 ++++++++++++++++---
 4 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl
index 3fc4d57ba..3e67290d6 100644
--- a/apps/emqx_bridge/src/emqx_bridge_app.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_app.erl
@@ -29,6 +29,7 @@
 
 start(_StartType, _StartArgs) ->
     {ok, Sup} = emqx_bridge_sup:start_link(),
+    ok = start_ee_apps(),
     ok = emqx_bridge:load(),
     ok = emqx_bridge:load_hook(),
     ok = emqx_config_handler:add_handler(?LEAF_NODE_HDLR_PATH, ?MODULE),
@@ -41,6 +42,16 @@ stop(_State) ->
     ok = emqx_bridge:unload_hook(),
     ok.
 
+-ifdef(EMQX_RELEASE_EDITION).
+start_ee_apps() ->
+    {ok, _} = application:ensure_all_started(emqx_ee_bridge),
+    {ok, _} = application:ensure_all_started(emqx_ee_connector),
+    ok.
+-else.
+start_ee_apps() ->
+    ok.
+-endif.
+
 %% NOTE: We depends on the `emqx_bridge:pre_config_update/3` to restart/stop the
 %%       underlying resources.
 pre_config_update(_, {_Oper, _, _}, undefined) ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index a94089c21..6b2987307 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -35,6 +35,6 @@ fields(bridges) ->
         {hstream,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
-                #{desc => <<"hstream_webhook">>}
+                #{desc => <<"emqx enterprise config">>}
             )}
     ].
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
index 42dd86f14..56bac456e 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
@@ -36,10 +36,10 @@ values(get) ->
 values(post) ->
     #{
         type => hstream,
-        name => <<"hstream_bridge_demo">>,
+        name => <<"demo">>,
         url => <<"http://127.0.0.1:6570">>,
         stream => <<"stream1">>,
-        ordering_key => <<"${topic}">>,
+        ordering_key => <<"some_key">>,
         pool_size => 8,
         enable => true,
         direction => egress,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
index 7ba92f809..fa8439851 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
@@ -46,9 +46,14 @@ on_stop(InstId, #{client := Client, producer := Producer}) ->
         stop_producer => StopProducerRes
     }).
 
-on_query(_InstId, {OrderingKey, Payload, Record}, AfterQuery, #{producer := Producer}) ->
-    Record = hstreamdb:to_record(OrderingKey, raw, Payload),
-    do_append(AfterQuery, false, Producer, Record).
+on_query(
+    _InstId,
+    {send_message, Data},
+    AfterQuery,
+    #{producer := Producer, ordering_key := OrderingKey, payload := Payload}
+) ->
+    Record = to_record(OrderingKey, Payload, Data),
+    do_append(AfterQuery, Producer, Record).
 
 on_get_status(_InstId, #{client := Client}) ->
     case is_alive(Client) of
@@ -88,8 +93,13 @@ start_client(InstId, Config) ->
         do_start_client(InstId, Config)
     catch
         E:R:S ->
-            io:format("E:R:S ~p:~p ~n~p~n", [E, R, S]),
-            error(E)
+            ?SLOG(error, #{
+                msg => "start hstream connector error",
+                connector => InstId,
+                error => E,
+                reason => R,
+                stack => S
+            })
     end.
 
 do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) ->
@@ -167,7 +177,18 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi
                 msg => "hstream connector: producer started"
             }),
             EnableBatch = maps:get(enable_batch, Options, false),
-            {ok, #{client => Client, producer => Producer, enable_batch => EnableBatch}};
+            PayloadBin = maps:get(payload, Options, <<"">>),
+            Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin),
+            OrderingKeyBin = maps:get(ordering_key, Options, <<"">>),
+            OrderingKey = emqx_plugin_libs_rule:preproc_tmpl(OrderingKeyBin),
+            State = #{
+                client => Client,
+                producer => Producer,
+                enable_batch => EnableBatch,
+                ordering_key => OrderingKey,
+                payload => Payload
+            },
+            {ok, State};
         {error, {already_started, Pid}} ->
             ?SLOG(info, #{
                 msg => "starting hstream connector: producer, find old producer. restart producer",
@@ -184,6 +205,19 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi
             {error, Reason}
     end.
 
+to_record(OrderingKeyTmpl, PayloadTmpl, Data) ->
+    OrderingKey = emqx_plugin_libs_rule:proc_tmpl(OrderingKeyTmpl, Data),
+    Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTmpl, Data),
+    to_record(OrderingKey, Payload).
+
+to_record(OrderingKey, Payload) when is_binary(OrderingKey) ->
+    to_record(binary_to_list(OrderingKey), Payload);
+to_record(OrderingKey, Payload) ->
+    hstreamdb:to_record(OrderingKey, raw, Payload).
+
+do_append(AfterQuery, Producer, Record) ->
+    do_append(AfterQuery, false, Producer, Record).
+
 %% TODO: this append is async, remove or change it after we have better disk cache.
 % do_append(AfterQuery, true, Producer, Record) ->
 %     case hstreamdb:append(Producer, Record) of
@@ -221,10 +255,10 @@ do_append(AfterQuery, false, Producer, Record) ->
     end.
 
 client_name(InstId) ->
-    "backend_hstream_client:" ++ to_string(InstId).
+    "client:" ++ to_string(InstId).
 
 produce_name(ActionId) ->
-    list_to_atom("backend_hstream_producer:" ++ to_string(ActionId)).
+    list_to_atom("producer:" ++ to_string(ActionId)).
 
 to_string(List) when is_list(List) -> List;
 to_string(Bin) when is_binary(Bin) -> binary_to_list(Bin);

From be6de4aad05d4381bd3ced762ed80d6fac38f84e Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Tue, 26 Jul 2022 15:54:58 +0800
Subject: [PATCH 005/232] fix: ee bridge & connector remove app mod

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      |  2 +-
 apps/emqx_bridge/src/emqx_bridge_app.erl      |  2 +-
 apps/emqx_bridge/src/emqx_bridge_resource.erl |  6 ++---
 apps/emqx_bridge/src/emqx_bridge_schema.erl   |  4 +--
 .../emqx_ee_bridge/src/emqx_ee_bridge.app.src |  2 --
 .../emqx_ee_bridge/src/emqx_ee_bridge_app.erl | 17 ------------
 .../emqx_ee_bridge/src/emqx_ee_bridge_sup.erl | 27 -------------------
 lib-ee/emqx_ee_connector/rebar.config         |  2 +-
 .../src/emqx_ee_connector.app.src             |  2 --
 .../src/emqx_ee_connector_app.erl             | 17 ------------
 .../src/emqx_ee_connector_sup.erl             | 27 -------------------
 11 files changed, 8 insertions(+), 100 deletions(-)
 delete mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl
 delete mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl
 delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl
 delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 0d3facc5c..e8eb91e57 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -183,7 +183,7 @@ conn_bridge_examples(Method) ->
     EE = ee_conn_bridge_examples(Method),
     maps:merge(Broker, EE).
 
--ifdef(EMQX_RELEASE_EDITION).
+-if(?EMQX_RELEASE_EDITION == ee).
 ee_conn_bridge_examples(Method) ->
     emqx_ee_bridge:conn_bridge_examples(Method).
 -else.
diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl
index 3e67290d6..cac6ab1e6 100644
--- a/apps/emqx_bridge/src/emqx_bridge_app.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_app.erl
@@ -42,7 +42,7 @@ stop(_State) ->
     ok = emqx_bridge:unload_hook(),
     ok.
 
--ifdef(EMQX_RELEASE_EDITION).
+-if(?EMQX_RELEASE_EDITION == ee).
 start_ee_apps() ->
     {ok, _} = application:ensure_all_started(emqx_ee_bridge),
     {ok, _} = application:ensure_all_started(emqx_ee_connector),
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index c82a290b3..1df6e2b91 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -42,7 +42,7 @@
     reset_metrics/1
 ]).
 
--ifdef(EMQX_RELEASE_EDITION).
+-if(?EMQX_RELEASE_EDITION == ee).
 bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt;
 bridge_to_resource_type(mqtt) -> emqx_connector_mqtt;
 bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http;
@@ -291,11 +291,11 @@ parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direc
 parse_confs(Type, Name, Conf) ->
     parse_enterprise_confs(Type, Name, Conf).
 
--ifdef(EMQX_RELEASE_EDITION).
+-if(?EMQX_RELEASE_EDITION == ee).
 parse_enterprise_confs(Type, Name, Conf) ->
     emqx_ee_bridge:parse_conf(Type, Name, Conf).
 -else.
-parse_enterprise_confs(Type, Name, Conf) ->
+parse_enterprise_confs(Type, Name, _Conf) ->
     error({not_supported, Type, Name}).
 -endif.
 
diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl
index de27cd34c..0b885db8c 100644
--- a/apps/emqx_bridge/src/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl
@@ -48,7 +48,7 @@ http_schema(Method) ->
     ExtSchemas = [ref(Module, Method) || Module <- schema_modules()],
     hoconsc:union(Schemas ++ ExtSchemas).
 
--ifdef(EMQX_RELEASE_EDITION).
+-if(?EMQX_RELEASE_EDITION == ee).
 schema_modules() ->
     [emqx_bridge_webhook_schema] ++ emqx_ee_bridge:schema_modules().
 -else.
@@ -158,7 +158,7 @@ fields("node_status") ->
         {"status", mk(status(), #{})}
     ].
 
--ifdef(EMQX_RELEASE_EDITION).
+-if(?EMQX_RELEASE_EDITION == ee).
 ee_fields_bridges() ->
     emqx_ee_bridge:fields(bridges).
 -else.
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index 58846577e..a578b7d0d 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -1,8 +1,6 @@
 {application, emqx_ee_bridge, [
-    {description, "An OTP application"},
     {vsn, "0.1.0"},
     {registered, []},
-    {mod, {emqx_ee_bridge_app, []}},
     {applications, [
         kernel,
         stdlib
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl
deleted file mode 100644
index 22268e285..000000000
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl
+++ /dev/null
@@ -1,17 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
-
--module(emqx_ee_bridge_app).
-
--behaviour(application).
-
--export([start/2, stop/1]).
-
-start(_StartType, _StartArgs) ->
-    emqx_ee_bridge_sup:start_link().
-
-stop(_State) ->
-    ok.
-
-%% internal functions
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl
deleted file mode 100644
index 5a2484442..000000000
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl
+++ /dev/null
@@ -1,27 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
-
--module(emqx_ee_bridge_sup).
-
--behaviour(supervisor).
-
--export([start_link/0]).
-
--export([init/1]).
-
--define(SERVER, ?MODULE).
-
-start_link() ->
-    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
-
-init([]) ->
-    SupFlags = #{
-        strategy => one_for_all,
-        intensity => 0,
-        period => 1
-    },
-    ChildSpecs = [],
-    {ok, {SupFlags, ChildSpecs}}.
-
-%% internal functions
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index 1f16e648e..38194cbf5 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -1,6 +1,6 @@
 {erl_opts, [debug_info]}.
 {deps, [
-  {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.4"}}}
+  {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}
 ]}.
 
 {shell, [
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 214488f80..6d9fea3c9 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,8 +1,6 @@
 {application, emqx_ee_connector, [
-    {description, "An OTP application"},
     {vsn, "0.1.0"},
     {registered, []},
-    {mod, {emqx_ee_connector_app, []}},
     {applications, [
         kernel,
         stdlib,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl
deleted file mode 100644
index 13db2ccb0..000000000
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl
+++ /dev/null
@@ -1,17 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
-
--module(emqx_ee_connector_app).
-
--behaviour(application).
-
--export([start/2, stop/1]).
-
-start(_StartType, _StartArgs) ->
-    emqx_ee_connector_sup:start_link().
-
-stop(_State) ->
-    ok.
-
-%% internal functions
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl
deleted file mode 100644
index f8a219fdc..000000000
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl
+++ /dev/null
@@ -1,27 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
-
--module(emqx_ee_connector_sup).
-
--behaviour(supervisor).
-
--export([start_link/0]).
-
--export([init/1]).
-
--define(SERVER, ?MODULE).
-
-start_link() ->
-    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
-
-init([]) ->
-    SupFlags = #{
-        strategy => one_for_all,
-        intensity => 0,
-        period => 1
-    },
-    ChildSpecs = [],
-    {ok, {SupFlags, ChildSpecs}}.
-
-%% internal functions

From 9ae7c6265609f638936b52b898a96da7e4d26177 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Wed, 27 Jul 2022 09:48:39 +0800
Subject: [PATCH 006/232] fix: exs deps & bad suites

---
 apps/emqx_bridge/src/emqx_bridge.app.src      |  2 +-
 apps/emqx_bridge/src/emqx_bridge_resource.erl | 20 ++++++-------------
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  6 +-----
 .../test/ee_bridge_hstream_SUITE.erl          | 16 +++++++++++++++
 .../i18n/emqx_ee_connector_hstream.conf       | 16 +++++++--------
 .../test/ee_connector_hstream_SUITE.erl       | 16 +++++++++++++++
 mix.exs                                       |  7 +++++--
 7 files changed, 53 insertions(+), 30 deletions(-)
 create mode 100644 lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl
 create mode 100644 lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl

diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src
index 70550efe4..fe19ed066 100644
--- a/apps/emqx_bridge/src/emqx_bridge.app.src
+++ b/apps/emqx_bridge/src/emqx_bridge.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "An OTP application"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index 1df6e2b91..cbd337970 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -239,7 +239,7 @@ is_tmp_path(TmpPath, File) ->
     string:str(str(File), str(TmpPath)) > 0.
 
 parse_confs(
-    webhook,
+    Type,
     _Name,
     #{
         url := Url,
@@ -248,7 +248,7 @@ parse_confs(
         headers := Headers,
         request_timeout := ReqTimeout
     } = Conf
-) ->
+) when Type == webhook orelse Type == <<"webhook">> ->
     {BaseUrl, Path} = parse_url(Url),
     {ok, BaseUrl2} = emqx_http_lib:uri_parse(BaseUrl),
     Conf#{
@@ -262,7 +262,7 @@ parse_confs(
                 request_timeout => ReqTimeout
             }
     };
-parse_confs(Type = mqtt, Name, #{connector := ConnId, direction := Direction} = Conf) when
+parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) when
     is_binary(ConnId)
 ->
     case emqx_connector:parse_connector_id(ConnId) of
@@ -278,7 +278,7 @@ parse_confs(Type = mqtt, Name, #{connector := ConnId, direction := Direction} =
         {_ConnType, _ConnName} ->
             error({cannot_use_connector_with_different_type, ConnId})
     end;
-parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when
+parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when
     is_map(ConnectorConfs)
 ->
     make_resource_confs(
@@ -288,16 +288,8 @@ parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direc
         Type,
         Name
     );
-parse_confs(Type, Name, Conf) ->
-    parse_enterprise_confs(Type, Name, Conf).
-
--if(?EMQX_RELEASE_EDITION == ee).
-parse_enterprise_confs(Type, Name, Conf) ->
-    emqx_ee_bridge:parse_conf(Type, Name, Conf).
--else.
-parse_enterprise_confs(Type, Name, _Conf) ->
-    error({not_supported, Type, Name}).
--endif.
+parse_confs(_Type, _Name, Conf) ->
+    Conf.
 
 make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
     BName = bridge_id(Type, Name),
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 6b2987307..7f051a158 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -9,7 +9,6 @@
     schema_modules/0,
     conn_bridge_examples/1,
     resource_type/1,
-    parse_conf/3,
     fields/1
 ]).
 
@@ -27,14 +26,11 @@ conn_bridge_examples(Method) ->
 resource_type(hstream) -> emqx_ee_connector_hstream;
 resource_type(<<"hstream">>) -> emqx_ee_connector_hstream.
 
-parse_conf(_Type, _Name, Conf) ->
-    Conf.
-
 fields(bridges) ->
     [
         {hstream,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
-                #{desc => <<"emqx enterprise config">>}
+                #{desc => <<"EMQX Enterprise Config">>}
             )}
     ].
diff --git a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl
new file mode 100644
index 000000000..d03c13cac
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl
@@ -0,0 +1,16 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(ee_bridge_hstream_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+%% TODO:
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
index 605944997..fce00baed 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
@@ -2,41 +2,41 @@
 emqx_ee_connector_hstream {
     url {
         desc {
-                en: """Hstream Server URL"""
+                en: """HStream Server URL"""
                 zh: """HStream 服务器 URL"""
             }
         label {
-                en: """Hstream Server URL"""
+                en: """HStream Server URL"""
                 zh: """HStream 服务器 URL"""
                 }
         }
     stream_name {
         desc {
-                en: """Hstream Stream Name"""
+                en: """HStream Stream Name"""
                 zh: """HStream 流名称"""
             }
         label {
-                en: """Hstream Stream Name"""
+                en: """HStream Stream Name"""
                 zh: """HStream 流名称"""
                 }
     }
     ordering_key {
         desc {
-                en: """Hstream Ordering Key"""
+                en: """HStream Ordering Key"""
                 zh: """HStream 分区键"""
             }
         label {
-                en: """Hstream Ordering Key"""
+                en: """HStream Ordering Key"""
                 zh: """HStream 分区键"""
                 }
     }
     pool_size {
         desc {
-                en: """Hstream Pool Size"""
+                en: """HStream Pool Size"""
                 zh: """HStream 连接池大小"""
             }
         label {
-                en: """Hstream Pool Size"""
+                en: """HStream Pool Size"""
                 zh: """HStream 连接池大小"""
             }
     }
diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl
new file mode 100644
index 000000000..cebe77c6a
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl
@@ -0,0 +1,16 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(ee_connector_hstream_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+%% TODO:
diff --git a/mix.exs b/mix.exs
index 714279cfd..2408b68f3 100644
--- a/mix.exs
+++ b/mix.exs
@@ -88,7 +88,8 @@ defmodule EMQXUmbrella.MixProject do
       {:ranch,
        github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
       # in conflict by grpc and eetcd
-      {:gpb, "4.11.2", override: true, runtime: false}
+      {:gpb, "4.11.2", override: true, runtime: false},
+      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}
     ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
   end
 
@@ -215,7 +216,9 @@ defmodule EMQXUmbrella.MixProject do
       if(edition_type == :enterprise,
         do: [
           emqx_license: :permanent,
-          emqx_enterprise_conf: :load
+          emqx_enterprise_conf: :load,
+          emqx_ee_connector: :permanent,
+          emqx_ee_bridge: :permanent
         ],
         else: []
       )

From a4992ef1b52b722b424bae8fe61aa8e22baf0686 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Thu, 28 Jul 2022 16:47:30 +0800
Subject: [PATCH 007/232] fix: hstreamdb connector conf & api

---
 apps/emqx_bridge/src/emqx_bridge_resource.erl |  4 +-
 .../emqx_connector/src/emqx_connector_api.erl | 16 +++-
 .../src/emqx_connector_schema.erl             | 24 +++++-
 .../i18n/emqx_ee_bridge_hstream.conf          | 10 +++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  6 +-
 .../src/emqx_ee_bridge_hstream.erl            | 40 +++++-----
 .../i18n/emqx_ee_connector_hstream.conf       | 21 +++++
 .../src/emqx_ee_connector.erl                 | 32 ++++++++
 .../src/emqx_ee_connector_hstream.erl         | 79 ++++++++++++++-----
 9 files changed, 179 insertions(+), 53 deletions(-)
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl

diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index cbd337970..6bb30e6b6 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -287,9 +287,7 @@ parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} =
         maps:without([connector, direction], Conf),
         Type,
         Name
-    );
-parse_confs(_Type, _Name, Conf) ->
-    Conf.
+    ).
 
 make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
     BName = bridge_id(Type, Name),
diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl
index 18409fafd..8dcd3a4aa 100644
--- a/apps/emqx_connector/src/emqx_connector_api.erl
+++ b/apps/emqx_connector/src/emqx_connector_api.erl
@@ -80,7 +80,7 @@ connector_info_array_example(Method) ->
     [Config || #{value := Config} <- maps:values(connector_info_examples(Method))].
 
 connector_info_examples(Method) ->
-    lists:foldl(
+    Fun =
         fun(Type, Acc) ->
             SType = atom_to_list(Type),
             maps:merge(Acc, #{
@@ -90,9 +90,17 @@ connector_info_examples(Method) ->
                 }
             })
         end,
-        #{},
-        ?CONN_TYPES
-    ).
+    Broker = lists:foldl(Fun, #{}, ?CONN_TYPES),
+    EE = ee_example(Method),
+    maps:merge(Broker, EE).
+
+-if(?EMQX_RELEASE_EDITION == ee).
+ee_example(Method) ->
+    emqx_ee_connector:connector_examples(Method).
+-else.
+ee_example(_Method) ->
+    #{}.
+-endif.
 
 info_example(Type, Method) ->
     maps:merge(
diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl
index b0f20924f..65d51443b 100644
--- a/apps/emqx_connector/src/emqx_connector_schema.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema.erl
@@ -44,7 +44,9 @@ post_request() ->
     http_schema("post").
 
 http_schema(Method) ->
-    Schemas = [?R_REF(schema_mod(Type), Method) || Type <- ?CONN_TYPES],
+    Broker = [?R_REF(schema_mod(Type), Method) || Type <- ?CONN_TYPES],
+    EE = [?R_REF(Module, Method) || Module <- schema_modules()],
+    Schemas = Broker ++ EE,
     ?UNION(Schemas).
 
 %%======================================================================================
@@ -57,13 +59,29 @@ roots() -> ["connectors"].
 fields(connectors) ->
     fields("connectors");
 fields("connectors") ->
-    [
+    Broker = [
         {mqtt,
             ?HOCON(
                 ?MAP(name, ?R_REF(emqx_connector_mqtt_schema, "connector")),
                 #{desc => ?DESC("mqtt")}
             )}
-    ].
+    ],
+    EE = ee_fields_connectors(),
+    Broker ++ EE.
+
+-if(?EMQX_RELEASE_EDITION == ee).
+schema_modules() ->
+    emqx_ee_connector:schema_modules().
+
+ee_fields_connectors() ->
+    emqx_ee_connector:fields(connectors).
+-else.
+ee_fields_connectors() ->
+    [].
+
+schema_modules() ->
+    [].
+-endif.
 
 desc(Record) when
     Record =:= connectors;
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
index a110dbf27..20b189293 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
@@ -81,4 +81,14 @@ will be forwarded.
             zh: "Bridge 名字"
         }
     }
+    desc_connector {
+        desc {
+                en: """Generic configuration for the connector."""
+                zh: """连接器的通用配置。"""
+        }
+        label: {
+                en: "Connector Generic Configuration"
+                zh: "连接器通用配置。"
+        }
+    }
 }
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 7f051a158..0ed8ee1fe 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -23,12 +23,12 @@ conn_bridge_examples(Method) ->
         end,
     lists:foldl(Fun, #{}, schema_modules()).
 
-resource_type(hstream) -> emqx_ee_connector_hstream;
-resource_type(<<"hstream">>) -> emqx_ee_connector_hstream.
+resource_type(hstreamdb) -> emqx_ee_connector_hstream;
+resource_type(<<"hstreamdb">>) -> emqx_ee_connector_hstream.
 
 fields(bridges) ->
     [
-        {hstream,
+        {hstreamdb,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
                 #{desc => <<"EMQX Enterprise Config">>}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
index 56bac456e..73f7a20eb 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
@@ -7,7 +7,7 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include("emqx_ee_bridge.hrl").
 
--import(hoconsc, [mk/2, enum/1]).
+-import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -export([
     conn_bridge_example/1
@@ -25,7 +25,7 @@
 
 conn_bridge_example(Method) ->
     #{
-        <<"hstream">> => #{
+        <<"hstreamdb">> => #{
             summary => <<"HStreamDB Bridge">>,
             value => values(Method)
         }
@@ -35,12 +35,9 @@ values(get) ->
     maps:merge(values(post), ?METRICS_EXAMPLE);
 values(post) ->
     #{
-        type => hstream,
+        type => hstreamdb,
         name => <<"demo">>,
-        url => <<"http://127.0.0.1:6570">>,
-        stream => <<"stream1">>,
-        ordering_key => <<"some_key">>,
-        pool_size => 8,
+        connector => <<"hstreamdb:connector">>,
         enable => true,
         direction => egress,
         local_topic => <<"local/topic/#">>,
@@ -56,11 +53,13 @@ namespace() -> "bridge".
 roots() -> [].
 
 fields("config") ->
-    ExtConfig = [
+    [
+        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
         {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
-        {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})}
-    ],
-    basic_config() ++ ExtConfig;
+        {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})},
+        {connector, field(connector)}
+    ];
 fields("post") ->
     [type_field(), name_field() | fields("config")];
 fields("put") ->
@@ -68,6 +67,16 @@ fields("put") ->
 fields("get") ->
     emqx_bridge_schema:metrics_status_fields() ++ fields("post").
 
+field(connector) ->
+    mk(
+        hoconsc:union([binary(), ref(emqx_ee_connector_hstream, config)]),
+        #{
+            required => true,
+            example => <<"hstreamdb:demo">>,
+            desc => ?DESC("desc_connector")
+        }
+    ).
+
 desc("config") ->
     ?DESC("desc_config");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
@@ -75,17 +84,10 @@ desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
 desc(_) ->
     undefined.
 
-basic_config() ->
-    Basic = [
-        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
-        {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}
-    ],
-    emqx_ee_connector_hstream:fields(config) ++ Basic.
-
 %% -------------------------------------------------------------------------------------------------
 %% internal
 type_field() ->
-    {type, mk(enum([hstream]), #{required => true, desc => ?DESC("desc_type")})}.
+    {type, mk(enum([hstreamdb]), #{required => true, desc => ?DESC("desc_type")})}.
 
 name_field() ->
     {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
index fce00baed..af9eda92a 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
@@ -1,5 +1,26 @@
 
 emqx_ee_connector_hstream {
+    type {
+        desc {
+                en: "The Connector Type."
+                zh: "连接器类型。"
+        }
+        label: {
+                en: "Connector Type"
+                zh: "连接器类型"
+            }
+    }
+
+    name {
+        desc {
+                en: "Connector name, used as a human-readable description of the connector."
+                zh: "连接器名称，人类可读的连接器描述。"
+        }
+        label: {
+                en: "Connector Name"
+                zh: "连接器名称"
+            }
+    }
     url {
         desc {
                 en: """HStream Server URL"""
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
new file mode 100644
index 000000000..7d175f0eb
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
@@ -0,0 +1,32 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_connector).
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-export([
+    schema_modules/0,
+    fields/1,
+    connector_examples/1
+]).
+
+schema_modules() ->
+    [emqx_ee_connector_hstream].
+
+fields(connectors) ->
+    [
+        {hstreamdb,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_connector_hstream, config)),
+                #{desc => <<"EMQX Enterprise Config">>}
+            )}
+    ].
+
+connector_examples(Method) ->
+    Fun =
+        fun(Module, Examples) ->
+            Example = erlang:apply(Module, connector_example, [Method]),
+            maps:merge(Examples, Example)
+        end,
+    lists:foldl(Fun, #{}, schema_modules()).
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
index fa8439851..00c7e6f61 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
@@ -25,7 +25,8 @@
 
 -export([
     roots/0,
-    fields/1
+    fields/1,
+    connector_example/1
 ]).
 
 %% -------------------------------------------------------------------------------------------------
@@ -38,7 +39,7 @@ on_stop(InstId, #{client := Client, producer := Producer}) ->
     StopClientRes = hstreamdb:stop_client(Client),
     StopProducerRes = hstreamdb:stop_producer(Producer),
     ?SLOG(info, #{
-        msg => "stop hstream connector",
+        msg => "stop hstreamdb connector",
         connector => InstId,
         client => Client,
         producer => Producer,
@@ -64,7 +65,7 @@ on_get_status(_InstId, #{client := Client}) ->
     end.
 
 %% -------------------------------------------------------------------------------------------------
-%% hstream batch callback
+%% hstreamdb batch callback
 %% TODO: maybe remove it after disk cache is ready
 
 on_flush_result({{flush, _Stream, _Records}, {ok, _Resp}}) ->
@@ -82,9 +83,41 @@ fields(config) ->
     [
         {url, mk(binary(), #{required => true, desc => ?DESC("url")})},
         {stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})},
-        {ordering_key, mk(binary(), #{required => true, desc => ?DESC("ordering_key")})},
+        {ordering_key, mk(binary(), #{required => false, desc => ?DESC("ordering_key")})},
         {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}
-    ].
+    ];
+fields("get") ->
+    fields("post");
+fields("put") ->
+    fields(config);
+fields("post") ->
+    [
+        {type, mk(hstreamdb, #{required => true, desc => ?DESC("type")})},
+        {name, mk(binary(), #{required => true, desc => ?DESC("name")})}
+    ] ++ fields("put").
+
+connector_example(Method) ->
+    #{
+        <<"hstreamdb">> => #{
+            summary => <<"HStreamDB Connector">>,
+            value => values(Method)
+        }
+    }.
+
+values(post) ->
+    maps:merge(values(put), #{name => <<"connector">>});
+values(get) ->
+    values(post);
+values(put) ->
+    #{
+        type => hstreamdb,
+        url => <<"http://127.0.0.1:6570">>,
+        stream => <<"stream1">>,
+        ordering_key => <<"some_key">>,
+        pool_size => 8
+    };
+values(_) ->
+    #{}.
 
 %% -------------------------------------------------------------------------------------------------
 %% internal functions
@@ -94,7 +127,7 @@ start_client(InstId, Config) ->
     catch
         E:R:S ->
             ?SLOG(error, #{
-                msg => "start hstream connector error",
+                msg => "start hstreamdb connector error",
                 connector => InstId,
                 error => E,
                 reason => R,
@@ -104,7 +137,7 @@ start_client(InstId, Config) ->
 
 do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) ->
     ?SLOG(info, #{
-        msg => "starting hstream connector: client",
+        msg => "starting hstreamdb connector: client",
         connector => InstId,
         config => Config
     }),
@@ -118,21 +151,21 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) ->
             case is_alive(Client) of
                 true ->
                     ?SLOG(info, #{
-                        msg => "hstream connector: client started",
+                        msg => "hstreamdb connector: client started",
                         connector => InstId,
                         client => Client
                     }),
                     start_producer(InstId, Client, Config);
                 _ ->
                     ?SLOG(error, #{
-                        msg => "hstream connector: client not alive",
+                        msg => "hstreamdb connector: client not alive",
                         connector => InstId
                     }),
                     {error, connect_failed}
             end;
         {error, {already_started, Pid}} ->
             ?SLOG(info, #{
-                msg => "starting hstream connector: client, find old client. restart client",
+                msg => "starting hstreamdb connector: client, find old client. restart client",
                 old_client_pid => Pid,
                 old_client_name => ClientName
             }),
@@ -140,7 +173,7 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) ->
             start_client(InstId, Config);
         {error, Error} ->
             ?SLOG(error, #{
-                msg => "hstream connector: client failed",
+                msg => "hstreamdb connector: client failed",
                 connector => InstId,
                 reason => Error
             }),
@@ -155,7 +188,11 @@ is_alive(Client) ->
             false
     end.
 
-start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSize}) ->
+start_producer(
+    InstId,
+    Client,
+    Options = #{stream := Stream, pool_size := PoolSize, egress := #{payload := PayloadBin}}
+) ->
     %% TODO: change these batch options after we have better disk cache.
     BatchSize = maps:get(batch_size, Options, 100),
     Interval = maps:get(batch_interval, Options, 1000),
@@ -168,16 +205,15 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi
     ],
     Name = produce_name(InstId),
     ?SLOG(info, #{
-        msg => "starting hstream connector: producer",
+        msg => "starting hstreamdb connector: producer",
         connector => InstId
     }),
     case hstreamdb:start_producer(Client, Name, ProducerOptions) of
         {ok, Producer} ->
             ?SLOG(info, #{
-                msg => "hstream connector: producer started"
+                msg => "hstreamdb connector: producer started"
             }),
             EnableBatch = maps:get(enable_batch, Options, false),
-            PayloadBin = maps:get(payload, Options, <<"">>),
             Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin),
             OrderingKeyBin = maps:get(ordering_key, Options, <<"">>),
             OrderingKey = emqx_plugin_libs_rule:preproc_tmpl(OrderingKeyBin),
@@ -191,7 +227,8 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi
             {ok, State};
         {error, {already_started, Pid}} ->
             ?SLOG(info, #{
-                msg => "starting hstream connector: producer, find old producer. restart producer",
+                msg =>
+                    "starting hstreamdb connector: producer, find old producer. restart producer",
                 old_producer_pid => Pid,
                 old_producer_name => Name
             }),
@@ -199,7 +236,7 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi
             start_producer(InstId, Client, Options);
         {error, Reason} ->
             ?SLOG(error, #{
-                msg => "starting hstream connector: producer, failed",
+                msg => "starting hstreamdb connector: producer, failed",
                 reason => Reason
             }),
             {error, Reason}
@@ -223,13 +260,13 @@ do_append(AfterQuery, Producer, Record) ->
 %     case hstreamdb:append(Producer, Record) of
 %         ok ->
 %             ?SLOG(debug, #{
-%                 msg => "hstream producer async append success",
+%                 msg => "hstreamdb producer async append success",
 %                 record => Record
 %             }),
 %             emqx_resource:query_success(AfterQuery);
 %         {error, Reason} ->
 %             ?SLOG(error, #{
-%                 msg => "hstream producer async append failed",
+%                 msg => "hstreamdb producer async append failed",
 %                 reason => Reason,
 %                 record => Record
 %             }),
@@ -241,13 +278,13 @@ do_append(AfterQuery, false, Producer, Record) ->
     case hstreamdb:append_flush(Producer, Record) of
         {ok, _} ->
             ?SLOG(debug, #{
-                msg => "hstream producer sync append success",
+                msg => "hstreamdb producer sync append success",
                 record => Record
             }),
             emqx_resource:query_success(AfterQuery);
         {error, Reason} ->
             ?SLOG(error, #{
-                msg => "hstream producer sync append failed",
+                msg => "hstreamdb producer sync append failed",
                 reason => Reason,
                 record => Record
             }),

From b7c245c5b0074b607456dc839503c69d8d2ca2cf Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Thu, 28 Jul 2022 16:50:34 +0800
Subject: [PATCH 008/232] fix: hstreamdb zh docs

---
 .../i18n/emqx_ee_bridge_hstream.conf          | 18 +++++------
 .../i18n/emqx_ee_connector_hstream.conf       | 32 +++++++++----------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
index 20b189293..ad07ad377 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
@@ -31,32 +31,32 @@ will be forwarded.
     config_enable {
         desc {
             en: """Enable or disable this bridge"""
-            zh: """启用/禁用 Bridge"""
+            zh: """启用/禁用桥接"""
         }
         label {
             en: "Enable Or Disable Bridge"
-            zh: "启用/禁用 Bridge"
+            zh: "启用/禁用桥接"
         }
         }
     config_direction {
         desc {
             en: """The direction of this bridge, MUST be 'egress'"""
-            zh: """Bridge 的方向， 必须是 egress"""
+            zh: """桥接的方向， 必须是 egress"""
         }
         label {
             en: "Bridge Direction"
-            zh: "Bridge 方向"
+            zh: "桥接方向"
         }
     }
 
     desc_config {
         desc {
             en: """Configuration for an HStreamDB bridge."""
-            zh: """HStreamDB Bridge 配置"""
+            zh: """HStreamDB 桥接配置"""
         }
         label: {
             en: "HStreamDB Bridge Configuration"
-            zh: "HStreamDB Bridge 配置"
+            zh: "HStreamDB 桥接配置"
         }
     }
 
@@ -67,18 +67,18 @@ will be forwarded.
         }
         label {
             en: "Bridge Type"
-            zh: "Bridge 类型"
+            zh: "桥接类型"
         }
     }
 
     desc_name {
         desc {
             en: """Bridge name, used as a human-readable description of the bridge."""
-            zh: """Bridge 名字，Bridge 的可读描述"""
+            zh: """桥接名字，可读描述"""
         }
         label {
             en: "Bridge Name"
-            zh: "Bridge 名字"
+            zh: "桥接名字"
         }
     }
     desc_connector {
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
index af9eda92a..080076065 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
@@ -23,42 +23,42 @@ emqx_ee_connector_hstream {
     }
     url {
         desc {
-                en: """HStream Server URL"""
-                zh: """HStream 服务器 URL"""
+                en: """HStreamDB Server URL"""
+                zh: """HStreamDB 服务器 URL"""
             }
         label {
-                en: """HStream Server URL"""
-                zh: """HStream 服务器 URL"""
+                en: """HStreamDB Server URL"""
+                zh: """HStreamDB 服务器 URL"""
                 }
         }
     stream_name {
         desc {
-                en: """HStream Stream Name"""
-                zh: """HStream 流名称"""
+                en: """HStreamDB Stream Name"""
+                zh: """HStreamDB 流名称"""
             }
         label {
-                en: """HStream Stream Name"""
-                zh: """HStream 流名称"""
+                en: """HStreamDB Stream Name"""
+                zh: """HStreamDB 流名称"""
                 }
     }
     ordering_key {
         desc {
-                en: """HStream Ordering Key"""
-                zh: """HStream 分区键"""
+                en: """HStreamDB Ordering Key"""
+                zh: """HStreamDB 分区键"""
             }
         label {
-                en: """HStream Ordering Key"""
-                zh: """HStream 分区键"""
+                en: """HStreamDB Ordering Key"""
+                zh: """HStreamDB 分区键"""
                 }
     }
     pool_size {
         desc {
-                en: """HStream Pool Size"""
-                zh: """HStream 连接池大小"""
+                en: """HStreamDB Pool Size"""
+                zh: """HStreamDB 连接池大小"""
             }
         label {
-                en: """HStream Pool Size"""
-                zh: """HStream 连接池大小"""
+                en: """HStreamDB Pool Size"""
+                zh: """HStreamDB 连接池大小"""
             }
     }
 }

From 01e9e8a0eb19ad25992cf6126904e7c6968047c5 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Fri, 29 Jul 2022 14:10:01 +0800
Subject: [PATCH 009/232] feat: add ee influxdb connector & bridge

---
 .../src/emqx_connector_schema.erl             |   8 +-
 .../i18n/emqx_ee_bridge_hstream.conf          |   8 +-
 .../i18n/emqx_ee_bridge_influxdb.conf         |  94 +++++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  10 +-
 .../src/emqx_ee_bridge_influxdb.erl           |  99 ++++++++
 .../i18n/emqx_ee_connector_influxdb.conf      | 165 ++++++++++++
 .../src/emqx_ee_connector.erl                 |  34 ++-
 .../src/emqx_ee_connector_hstream.erl         |  16 +-
 .../src/emqx_ee_connector_influxdb.erl        | 240 ++++++++++++++++++
 9 files changed, 650 insertions(+), 24 deletions(-)
 create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
 create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl

diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl
index 65d51443b..f0c9479de 100644
--- a/apps/emqx_connector/src/emqx_connector_schema.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema.erl
@@ -45,7 +45,7 @@ post_request() ->
 
 http_schema(Method) ->
     Broker = [?R_REF(schema_mod(Type), Method) || Type <- ?CONN_TYPES],
-    EE = [?R_REF(Module, Method) || Module <- schema_modules()],
+    EE = ee_schemas(Method),
     Schemas = Broker ++ EE,
     ?UNION(Schemas).
 
@@ -70,8 +70,8 @@ fields("connectors") ->
     Broker ++ EE.
 
 -if(?EMQX_RELEASE_EDITION == ee).
-schema_modules() ->
-    emqx_ee_connector:schema_modules().
+ee_schemas(Method) ->
+    emqx_ee_connector:api_schemas(Method).
 
 ee_fields_connectors() ->
     emqx_ee_connector:fields(connectors).
@@ -79,7 +79,7 @@ ee_fields_connectors() ->
 ee_fields_connectors() ->
     [].
 
-schema_modules() ->
+ee_schemas(_) ->
     [].
 -endif.
 
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
index ad07ad377..2e4397a04 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
@@ -13,10 +13,10 @@ will be forwarded.
 注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。
 """
         }
-    label {
-            en: "Local Topic"
-            zh: "本地 Topic"
-        }
+        label {
+                en: "Local Topic"
+                zh: "本地 Topic"
+            }
     }
     payload {
         desc {
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
new file mode 100644
index 000000000..f421fb912
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -0,0 +1,94 @@
+emqx_ee_bridge_influxdb {
+    local_topic {
+        desc {
+            en: """
+The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic
+matching the local_topic will be forwarded.</br>
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is
+configured, then both the data got from the rule and the MQTT messages that match local_topic
+will be forwarded.
+"""
+            zh: """
+发送到 'local_topic' 的消息都会转发到 InfluxDB。 </br>
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。
+"""
+        }
+        label {
+                en: "Local Topic"
+                zh: "本地 Topic"
+            }
+    }
+    payload {
+        desc {
+            en: """The payload to be forwarded to the InfluxDB. Placeholders supported."""
+            zh: """要转发到 InfluxDB 的数据内容，支持占位符"""
+            }
+        label {
+            en: "Payload"
+            zh: "消息内容"
+        }
+    }
+    config_enable {
+        desc {
+            en: """Enable or disable this bridge"""
+            zh: """启用/禁用桥接"""
+        }
+        label {
+            en: "Enable Or Disable Bridge"
+            zh: "启用/禁用桥接"
+        }
+        }
+    config_direction {
+        desc {
+            en: """The direction of this bridge, MUST be 'egress'"""
+            zh: """桥接的方向， 必须是 egress"""
+        }
+        label {
+            en: "Bridge Direction"
+            zh: "桥接方向"
+        }
+    }
+
+    desc_config {
+        desc {
+            en: """Configuration for an InfluxDB bridge."""
+            zh: """InfluxDB 桥接配置"""
+        }
+        label: {
+            en: "InfluxDB Bridge Configuration"
+            zh: "InfluxDB 桥接配置"
+        }
+    }
+
+    desc_type {
+        desc {
+            en: """The Bridge Type"""
+            zh: """Bridge 类型"""
+        }
+        label {
+            en: "Bridge Type"
+            zh: "桥接类型"
+        }
+    }
+
+    desc_name {
+        desc {
+            en: """Bridge name, used as a human-readable description of the bridge."""
+            zh: """桥接名字，可读描述"""
+        }
+        label {
+            en: "Bridge Name"
+            zh: "桥接名字"
+        }
+    }
+    desc_connector {
+        desc {
+                en: """Generic configuration for the connector."""
+                zh: """连接器的通用配置。"""
+        }
+        label: {
+                en: "Connector Generic Configuration"
+                zh: "连接器通用配置。"
+        }
+    }
+}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 0ed8ee1fe..ff9661cc8 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -13,7 +13,7 @@
 ]).
 
 schema_modules() ->
-    [emqx_ee_bridge_hstream].
+    [emqx_ee_bridge_hstream, emqx_ee_bridge_influxdb].
 
 conn_bridge_examples(Method) ->
     Fun =
@@ -23,8 +23,9 @@ conn_bridge_examples(Method) ->
         end,
     lists:foldl(Fun, #{}, schema_modules()).
 
+resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8));
 resource_type(hstreamdb) -> emqx_ee_connector_hstream;
-resource_type(<<"hstreamdb">>) -> emqx_ee_connector_hstream.
+resource_type(influxdb) -> emqx_ee_connector_influxdb.
 
 fields(bridges) ->
     [
@@ -32,5 +33,10 @@ fields(bridges) ->
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
                 #{desc => <<"EMQX Enterprise Config">>}
+            )},
+        {influxdb,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_bridge_influxdb, "config")),
+                #{desc => <<"EMQX Enterprise Config">>}
             )}
     ].
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
new file mode 100644
index 000000000..d285c2621
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -0,0 +1,99 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_influxdb).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+-include("emqx_ee_bridge.hrl").
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-export([
+    conn_bridge_example/1
+]).
+
+-export([
+    namespace/0,
+    roots/0,
+    fields/1,
+    desc/1
+]).
+
+%% -------------------------------------------------------------------------------------------------
+%% api
+
+conn_bridge_example(Method) ->
+    #{
+        <<"influxdb">> => #{
+            summary => <<"InfluxDB Bridge">>,
+            value => values(Method)
+        }
+    }.
+
+values(get) ->
+    maps:merge(values(post), ?METRICS_EXAMPLE);
+values(post) ->
+    #{
+        type => influxdb,
+        name => <<"demo">>,
+        connector => <<"influxdb:api_v2_connector">>,
+        enable => true,
+        direction => egress,
+        local_topic => <<"local/topic/#">>,
+        payload => <<"${payload}">>
+    };
+values(put) ->
+    values(post).
+
+%% -------------------------------------------------------------------------------------------------
+%% Hocon Schema Definitions
+namespace() -> "bridge".
+
+roots() -> [].
+
+fields("config") ->
+    [
+        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
+        {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
+        {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})},
+        {connector, field(connector)}
+    ];
+fields("post") ->
+    [type_field(), name_field() | fields("config")];
+fields("put") ->
+    fields("config");
+fields("get") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields("post").
+
+field(connector) ->
+    ConnectorConfigRef =
+        [
+            ref(emqx_ee_connector_influxdb, udp),
+            ref(emqx_ee_connector_influxdb, api_v1),
+            ref(emqx_ee_connector_influxdb, api_v2)
+        ],
+    mk(
+        hoconsc:union([binary() | ConnectorConfigRef]),
+        #{
+            required => true,
+            example => <<"influxdb:demo">>,
+            desc => ?DESC("desc_connector")
+        }
+    ).
+
+desc("config") ->
+    ?DESC("desc_config");
+desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
+    ["Configuration for HStream using `", string:to_upper(Method), "` method."];
+desc(_) ->
+    undefined.
+
+%% -------------------------------------------------------------------------------------------------
+%% internal
+type_field() ->
+    {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})}.
+
+name_field() ->
+    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
new file mode 100644
index 000000000..a909c9a72
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
@@ -0,0 +1,165 @@
+
+emqx_ee_connector_influxdb {
+    type {
+        desc {
+                en: """The Connector Type."""
+                zh: """连接器类型。"""
+        }
+        label: {
+                en: """Connector Type"""
+                zh: """连接器类型"""
+            }
+    }
+
+    name {
+        desc {
+                en: """Connector name, used as a human-readable description of the connector."""
+                zh: """连接器名称，人类可读的连接器描述。"""
+        }
+        label: {
+                en: """Connector Name"""
+                zh: """连接器名称"""
+            }
+    }
+    host {
+        desc {
+                en: """InfluxDB host."""
+                zh: """InfluxDB 主机地址。"""
+        }
+        label: {
+                en: """Host"""
+                zh: """主机"""
+            }
+    }
+    port {
+        desc {
+                en: """InfluxDB port."""
+                zh: """InfluxDB 端口。"""
+        }
+        label: {
+                en: """Port"""
+                zh: """端口"""
+            }
+    }
+    protocol {
+        desc {
+                en: """InfluxDB protocol. UDP or HTTP API or HTTP API V2"""
+                zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2"""
+        }
+        label: {
+                en: """Protocol"""
+                zh: """协议"""
+            }
+    }
+    protocol_udp {
+        desc {
+                en: """InfluxDB protocol."""
+                zh: """InfluxDB UDP 协议"""
+        }
+        label: {
+                en: """UDP Protocol"""
+                zh: """UDP 协议"""
+            }
+    }
+    protocol_api_v1 {
+        desc {
+                en: """InfluxDB protocol. Support InfluxDB v1.8 and before."""
+                zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本"""
+        }
+        label: {
+                en: """HTTP API Protocol"""
+                zh: """HTTP API 协议"""
+            }
+    }
+    protocol_api_v2 {
+        desc {
+                en: """InfluxDB protocol. Support InfluxDB v2.0 and after."""
+                zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本"""
+        }
+        label: {
+                en: """HTTP API V2 Protocol"""
+                zh: """HTTP API V2 协议"""
+            }
+    }
+    database {
+        desc {
+                en: """InfluxDB database."""
+                zh: """InfluxDB 数据库。"""
+        }
+        label: {
+                en: "Database"
+                zh: "数据库"
+            }
+    }
+    username {
+        desc {
+                en: "InfluxDB username."
+                zh: "InfluxDB 用户名。"
+        }
+        label: {
+                en: "Username"
+                zh: "用户名"
+            }
+    }
+    password {
+        desc {
+                en: "InfluxDB password."
+                zh: "InfluxDB 密码。"
+        }
+        label: {
+                en: "Password"
+                zh: "密码"
+            }
+    }
+    bucket {
+        desc {
+                en: "InfluxDB bucket name."
+                zh: "InfluxDB bucket 名称"
+        }
+        label: {
+                en: "Bucket"
+                zh: "Bucket"
+            }
+    }
+    org {
+        desc {
+                en: """InfluxDB organization name."""
+                zh: """InfluxDB 组织名称。"""
+        }
+        label: {
+                en: """Organization"""
+                zh: """组织"""
+            }
+    }
+    token {
+        desc {
+                en: """InfluxDB token."""
+                zh: """InfluxDB token。"""
+        }
+        label: {
+                en: """Token"""
+                zh: """Token"""
+            }
+    }
+    precision {
+        desc {
+                en: """InfluxDB time precision."""
+                zh: """InfluxDB 时间精度。"""
+        }
+        label: {
+                en: """Time Precision"""
+                zh: """时间精度"""
+            }
+    }
+    pool_size {
+        desc {
+                en: """InfluxDB Pool Size"""
+                zh: """InfluxDB 连接池大小"""
+            }
+        label {
+                en: """InfluxDB Pool Size"""
+                zh: """InfluxDB 连接池大小"""
+            }
+    }
+
+}
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
index 7d175f0eb..d0884f945 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
@@ -6,13 +6,18 @@
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -export([
-    schema_modules/0,
     fields/1,
-    connector_examples/1
+    connector_examples/1,
+    api_schemas/1
 ]).
 
-schema_modules() ->
-    [emqx_ee_connector_hstream].
+api_schemas(Method) ->
+    [
+        ref(emqx_ee_connector_hstream, Method),
+        ref(emqx_ee_connector_influxdb, Method ++ "_udp"),
+        ref(emqx_ee_connector_influxdb, Method ++ "_api_v1"),
+        ref(emqx_ee_connector_influxdb, Method ++ "_api_v2")
+    ].
 
 fields(connectors) ->
     [
@@ -21,12 +26,27 @@ fields(connectors) ->
                 hoconsc:map(name, ref(emqx_ee_connector_hstream, config)),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
+    ] ++ fields(influxdb);
+fields(influxdb) ->
+    [
+        {Protocol,
+            mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, Protocol)), #{
+                desc => <<"EMQX Enterprise Config">>
+            })}
+     || Protocol <- [udp, api_v1, api_v2]
     ].
 
 connector_examples(Method) ->
-    Fun =
-        fun(Module, Examples) ->
-            Example = erlang:apply(Module, connector_example, [Method]),
+    MergeFun =
+        fun(Example, Examples) ->
             maps:merge(Examples, Example)
         end,
+    Fun =
+        fun(Module, Examples) ->
+            ConnectorExamples = erlang:apply(Module, connector_examples, [Method]),
+            lists:foldl(MergeFun, Examples, ConnectorExamples)
+        end,
     lists:foldl(Fun, #{}, schema_modules()).
+
+schema_modules() ->
+    [emqx_ee_connector_hstream, emqx_ee_connector_influxdb].
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
index 00c7e6f61..7a8ce3fe9 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
@@ -26,7 +26,7 @@
 -export([
     roots/0,
     fields/1,
-    connector_example/1
+    connector_examples/1
 ]).
 
 %% -------------------------------------------------------------------------------------------------
@@ -96,13 +96,15 @@ fields("post") ->
         {name, mk(binary(), #{required => true, desc => ?DESC("name")})}
     ] ++ fields("put").
 
-connector_example(Method) ->
-    #{
-        <<"hstreamdb">> => #{
-            summary => <<"HStreamDB Connector">>,
-            value => values(Method)
+connector_examples(Method) ->
+    [
+        #{
+            <<"hstreamdb">> => #{
+                summary => <<"HStreamDB Connector">>,
+                value => values(Method)
+            }
         }
-    }.
+    ].
 
 values(post) ->
     maps:merge(values(put), #{name => <<"connector">>});
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
new file mode 100644
index 000000000..0ed7964b7
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -0,0 +1,240 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_connector_influxdb).
+
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-behaviour(emqx_resource).
+
+-define(PUT_FIELDS_FILTER, fun({Name, _}) -> not lists:member(Name, [type, name]) end).
+
+%% callbacks of behaviour emqx_resource
+-export([
+    on_start/2,
+    on_stop/2,
+    on_query/4,
+    on_get_status/2
+]).
+
+-export([
+    fields/1,
+    connector_examples/1
+]).
+
+%% -------------------------------------------------------------------------------------------------
+%% resource callback
+
+on_start(InstId, Config) ->
+    start_client(InstId, Config).
+
+on_stop(_InstId, _State) ->
+    ok.
+
+on_query(_InstId, {send_message, _Data}, _AfterQuery, _State) ->
+    ok.
+
+on_get_status(_InstId, _State) ->
+    % connected;
+    disconnected.
+
+%% -------------------------------------------------------------------------------------------------
+%% schema
+
+fields("put_udp") ->
+    lists:filter(?PUT_FIELDS_FILTER, fields(udp));
+fields("put_api_v1") ->
+    lists:filter(?PUT_FIELDS_FILTER, fields(api_v1));
+fields("put_api_v2") ->
+    lists:filter(?PUT_FIELDS_FILTER, fields(api_v2));
+fields("get_udp") ->
+    fields(udp);
+fields("get_api_v1") ->
+    fields(api_v1);
+fields("get_api_v2") ->
+    fields(api_v2);
+fields("post_udp") ->
+    fields(udp);
+fields("post_api_v1") ->
+    fields(api_v1);
+fields("post_api_v2") ->
+    fields(api_v2);
+fields(basic) ->
+    [
+        {host,
+            mk(binary(), #{required => true, default => <<"120.0.0.1">>, desc => ?DESC("host")})},
+        {port, mk(pos_integer(), #{required => true, default => 8086, desc => ?DESC("port")})},
+        {precision,
+            mk(enum([ns, us, ms, s, m, h]), #{
+                required => false, default => ms, desc => ?DESC("precision")
+            })},
+        {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})},
+        {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("type")})},
+        {name, mk(binary(), #{required => true, desc => ?DESC("name")})}
+    ];
+fields(udp) ->
+    [
+        {protocol, mk(enum([udp]), #{required => true, desc => ?DESC("protocol_udp")})}
+    ] ++ fields(basic);
+fields(api_v1) ->
+    [
+        {protocol, mk(enum([api_v1]), #{required => true, desc => ?DESC("protocol_api_v1")})},
+        {database, mk(binary(), #{required => true, desc => ?DESC("database")})},
+        {username, mk(binary(), #{required => true, desc => ?DESC("username")})},
+        {password, mk(binary(), #{required => true, desc => ?DESC("password")})}
+    ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic);
+fields(api_v2) ->
+    [
+        {protocol, mk(enum([api_v2]), #{required => true, desc => ?DESC("protocol_api_v2")})},
+        {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})},
+        {org, mk(binary(), #{required => true, desc => ?DESC("org")})},
+        {token, mk(binary(), #{required => true, desc => ?DESC("token")})}
+    ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic).
+
+connector_examples(Method) ->
+    [
+        #{
+            <<"influxdb_udp">> => #{
+                summary => <<"InfluxDB UDP Connector">>,
+                value => values(udp, Method)
+            }
+        },
+        #{
+            <<"influxdb_api_v1">> => #{
+                summary => <<"InfluxDB HTTP API V1 Connector">>,
+                value => values(api_v1, Method)
+            }
+        },
+        #{
+            <<"influxdb_api_v2">> => #{
+                summary => <<"InfluxDB HTTP API V2 Connector">>,
+                value => values(api_v2, Method)
+            }
+        }
+    ].
+
+values(Protocol, get) ->
+    values(Protocol, post);
+values(Protocol, post) ->
+    ConnectorName = list_to_binary(io_lib:format("~p_connector", [Protocol])),
+    maps:merge(values(Protocol, put), #{type => influxdb, name => ConnectorName});
+values(udp, put) ->
+    #{
+        protocol => udp,
+        host => <<"127.0.0.1">>,
+        port => 8089,
+        precision => ms,
+        pool_size => 8
+    };
+values(api_v1, put) ->
+    #{
+        protocol => api_v1,
+        host => <<"127.0.0.1">>,
+        port => 8086,
+        precision => ms,
+        pool_size => 8,
+        database => <<"my_db">>,
+        username => <<"my_user">>,
+        password => <<"my_password">>,
+        ssl => #{enable => false}
+    };
+values(api_v2, put) ->
+    #{
+        protocol => api_v2,
+        host => <<"127.0.0.1">>,
+        port => 8086,
+        precision => ms,
+        pool_size => 8,
+        bucket => <<"my_bucket">>,
+        org => <<"my_org">>,
+        token => <<"my_token">>,
+        ssl => #{enable => false}
+    }.
+%% -------------------------------------------------------------------------------------------------
+%% internal functions
+
+start_client(InstId, Config) ->
+    io:format("InstId ~p~n", [InstId]),
+    client_config(InstId, Config).
+
+% ClientConfig = client_config(InstId, Config),
+% case influxdb:start_client(ClientConfig) of
+%     {ok, Client} ->
+%         true = influxdb:is_alive(Client),
+%         maybe_pool_size(Client, Params);
+%     {error, {already_started, Client0}} ->
+%         _ = influxdb:stop_client(Client0),
+%         {ok, Client} = influxdb:start_client(Options),
+%         true = influxdb:is_alive(Client),
+%         maybe_pool_size(Client, Params);
+%     {error, Reason} ->
+%         logger:log(error, "Initiate influxdb failed ~0p", [Reason]),
+%         error({start_pool_failed, ResId})
+% end.
+
+client_config(
+    _InstId,
+    Config = #{
+        host := Host,
+        port := Port,
+        pool_size := PoolSize
+    }
+) ->
+    [
+        {host, Host},
+        {port, Port},
+        {pool_size, PoolSize},
+        {pool, atom_pname_todo},
+        {precision, maps:get(precision, Config, ms)}
+    ] ++ protocol_config(Config).
+
+protocol_config(#{
+    protocol := udp
+}) ->
+    [
+        {protocol, udp}
+    ];
+protocol_config(#{
+    protocol := api_v1,
+    username := Username,
+    password := Password,
+    database := DB,
+    ssl := SSL
+}) ->
+    [
+        {protocol, http},
+        {version, v1},
+        {username, Username},
+        {password, Password},
+        {database, DB},
+        {ssl, SSL}
+    ] ++ ssl_config(SSL);
+protocol_config(#{
+    protocol := api_v2,
+    bucket := Bucket,
+    org := Org,
+    token := Token,
+    ssl := SSL
+}) ->
+    [
+        {protocol, http},
+        {version, v2},
+        {bucket, Bucket},
+        {org, Org},
+        {token, Token},
+        {ssl, SSL}
+    ] ++ ssl_config(SSL).
+
+ssl_config(#{enable := false}) ->
+    [
+        {https_enabled, false}
+    ];
+ssl_config(SSL = #{enable := true}) ->
+    [
+        {https_enabled, true},
+        {transport, ssl}
+    ] ++ maps:to_list(maps:remove(enable, SSL)).

From 4c7ca2217ce53d5bb94e71b0b4b72e139d27687a Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Fri, 29 Jul 2022 18:36:48 +0800
Subject: [PATCH 010/232] fix: add influxdb udp api_v1 api_v2 connector

---
 .../src/emqx_ee_bridge_influxdb.erl           |   6 +-
 lib-ee/emqx_ee_connector/rebar.config         |   3 +-
 .../src/emqx_ee_connector.erl                 |  13 +-
 .../src/emqx_ee_connector_influxdb.erl        | 124 ++++++++++++------
 mix.exs                                       |   6 +
 5 files changed, 102 insertions(+), 50 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index d285c2621..7004b70a1 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -70,9 +70,9 @@ fields("get") ->
 field(connector) ->
     ConnectorConfigRef =
         [
-            ref(emqx_ee_connector_influxdb, udp),
-            ref(emqx_ee_connector_influxdb, api_v1),
-            ref(emqx_ee_connector_influxdb, api_v2)
+            ref(emqx_ee_connector_influxdb, influxdb_udp),
+            ref(emqx_ee_connector_influxdb, influxdb_api_v1),
+            ref(emqx_ee_connector_influxdb, influxdb_api_v2)
         ],
     mk(
         hoconsc:union([binary() | ConnectorConfigRef]),
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index 38194cbf5..0a3f6866b 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -1,6 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, [
-  {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}
+  {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
+  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.2"}}}
 ]}.
 
 {shell, [
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
index d0884f945..55ccc2c2c 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
@@ -26,14 +26,17 @@ fields(connectors) ->
                 hoconsc:map(name, ref(emqx_ee_connector_hstream, config)),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
-    ] ++ fields(influxdb);
+    ];
+% ] ++ fields(influxdb);
 fields(influxdb) ->
     [
-        {Protocol,
-            mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, Protocol)), #{
+        {
+            influxdb,
+            mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, influxdb_udp)), #{
                 desc => <<"EMQX Enterprise Config">>
-            })}
-     || Protocol <- [udp, api_v1, api_v2]
+            })
+        }
+        %  || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2]
     ].
 
 connector_examples(Method) ->
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 0ed7964b7..1efc0b263 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -32,37 +32,41 @@
 on_start(InstId, Config) ->
     start_client(InstId, Config).
 
-on_stop(_InstId, _State) ->
-    ok.
+on_stop(_InstId, #{client := Client}) ->
+    influxdb:stop_client(Client).
 
 on_query(_InstId, {send_message, _Data}, _AfterQuery, _State) ->
     ok.
 
-on_get_status(_InstId, _State) ->
-    % connected;
-    disconnected.
+on_get_status(_InstId, #{client := Client}) ->
+    case influxdb:is_alive(Client) of
+        true ->
+            connected;
+        false ->
+            disconnected
+    end.
 
 %% -------------------------------------------------------------------------------------------------
 %% schema
 
 fields("put_udp") ->
-    lists:filter(?PUT_FIELDS_FILTER, fields(udp));
+    lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_udp));
 fields("put_api_v1") ->
-    lists:filter(?PUT_FIELDS_FILTER, fields(api_v1));
+    lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v1));
 fields("put_api_v2") ->
-    lists:filter(?PUT_FIELDS_FILTER, fields(api_v2));
+    lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v2));
 fields("get_udp") ->
-    fields(udp);
+    fields(influxdb_udp);
 fields("get_api_v1") ->
-    fields(api_v1);
+    fields(influxdb_api_v1);
 fields("get_api_v2") ->
-    fields(api_v2);
+    fields(influxdb_api_v2);
 fields("post_udp") ->
-    fields(udp);
+    fields(influxdb_udp);
 fields("post_api_v1") ->
-    fields(api_v1);
+    fields(influxdb_api_v1);
 fields("post_api_v2") ->
-    fields(api_v2);
+    fields(influxdb_api_v2);
 fields(basic) ->
     [
         {host,
@@ -73,23 +77,22 @@ fields(basic) ->
                 required => false, default => ms, desc => ?DESC("precision")
             })},
         {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})},
-        {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("type")})},
         {name, mk(binary(), #{required => true, desc => ?DESC("name")})}
     ];
-fields(udp) ->
+fields(influxdb_udp) ->
     [
-        {protocol, mk(enum([udp]), #{required => true, desc => ?DESC("protocol_udp")})}
+        {type, mk(influxdb_udp, #{required => true, desc => ?DESC("type")})}
     ] ++ fields(basic);
-fields(api_v1) ->
+fields(influxdb_api_v1) ->
     [
-        {protocol, mk(enum([api_v1]), #{required => true, desc => ?DESC("protocol_api_v1")})},
+        {type, mk(influxdb_api_v1, #{required => true, desc => ?DESC("type")})},
         {database, mk(binary(), #{required => true, desc => ?DESC("database")})},
         {username, mk(binary(), #{required => true, desc => ?DESC("username")})},
         {password, mk(binary(), #{required => true, desc => ?DESC("password")})}
     ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic);
-fields(api_v2) ->
+fields(influxdb_api_v2) ->
     [
-        {protocol, mk(enum([api_v2]), #{required => true, desc => ?DESC("protocol_api_v2")})},
+        {type, mk(influxdb_api_v2, #{required => true, desc => ?DESC("type")})},
         {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})},
         {org, mk(binary(), #{required => true, desc => ?DESC("org")})},
         {token, mk(binary(), #{required => true, desc => ?DESC("token")})}
@@ -120,11 +123,11 @@ connector_examples(Method) ->
 values(Protocol, get) ->
     values(Protocol, post);
 values(Protocol, post) ->
+    Type = list_to_atom(io_lib:format("influxdb_~p", [Protocol])),
     ConnectorName = list_to_binary(io_lib:format("~p_connector", [Protocol])),
-    maps:merge(values(Protocol, put), #{type => influxdb, name => ConnectorName});
+    maps:merge(values(Protocol, put), #{type => Type, name => ConnectorName});
 values(udp, put) ->
     #{
-        protocol => udp,
         host => <<"127.0.0.1">>,
         port => 8089,
         precision => ms,
@@ -132,7 +135,6 @@ values(udp, put) ->
     };
 values(api_v1, put) ->
     #{
-        protocol => api_v1,
         host => <<"127.0.0.1">>,
         port => 8086,
         precision => ms,
@@ -144,7 +146,6 @@ values(api_v1, put) ->
     };
 values(api_v2, put) ->
     #{
-        protocol => api_v2,
         host => <<"127.0.0.1">>,
         port => 8086,
         precision => ms,
@@ -158,23 +159,64 @@ values(api_v2, put) ->
 %% internal functions
 
 start_client(InstId, Config) ->
-    io:format("InstId ~p~n", [InstId]),
-    client_config(InstId, Config).
+    ClientConfig = client_config(InstId, Config),
+    ?SLOG(info, #{
+        msg => "starting influxdb connector",
+        connector => InstId,
+        config => Config,
+        client_config => ClientConfig
+    }),
+    try
+        do_start_client(InstId, ClientConfig, Config)
+    catch
+        E:R:S ->
+            ?SLOG(error, #{
+                msg => "start influxdb connector error",
+                connector => InstId,
+                error => E,
+                reason => R,
+                stack => S
+            }),
+            {error, R}
+    end.
 
-% ClientConfig = client_config(InstId, Config),
-% case influxdb:start_client(ClientConfig) of
-%     {ok, Client} ->
-%         true = influxdb:is_alive(Client),
-%         maybe_pool_size(Client, Params);
-%     {error, {already_started, Client0}} ->
-%         _ = influxdb:stop_client(Client0),
-%         {ok, Client} = influxdb:start_client(Options),
-%         true = influxdb:is_alive(Client),
-%         maybe_pool_size(Client, Params);
-%     {error, Reason} ->
-%         logger:log(error, "Initiate influxdb failed ~0p", [Reason]),
-%         error({start_pool_failed, ResId})
-% end.
+do_start_client(InstId, ClientConfig, Config = #{egress := #{payload := PayloadBin}}) ->
+    case influxdb:start_client(ClientConfig) of
+        {ok, Client} ->
+            case influxdb:is_alive(Client) of
+                true ->
+                    Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin),
+                    ?SLOG(info, #{
+                        msg => "starting influxdb connector success",
+                        connector => InstId,
+                        client => Client
+                    }),
+                    #{client => Client, payload => Payload};
+                false ->
+                    ?SLOG(error, #{
+                        msg => "starting influxdb connector failed",
+                        connector => InstId,
+                        client => Client,
+                        reason => "client is not alive"
+                    }),
+                    {error, influxdb_client_not_alive}
+            end;
+        {error, {already_started, Client0}} ->
+            ?SLOG(info, #{
+                msg => "starting influxdb connector,find already started client",
+                connector => InstId,
+                old_client => Client0
+            }),
+            _ = influxdb:stop_client(Client0),
+            do_start_client(InstId, ClientConfig, Config);
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "starting influxdb connector failed",
+                connector => InstId,
+                reason => Reason
+            }),
+            {error, Reason}
+    end.
 
 client_config(
     _InstId,
diff --git a/mix.exs b/mix.exs
index 6648aa35e..039172eb9 100644
--- a/mix.exs
+++ b/mix.exs
@@ -89,9 +89,15 @@ defmodule EMQXUmbrella.MixProject do
        github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
       # in conflict by grpc and eetcd
       {:gpb, "4.11.2", override: true, runtime: false},
+<<<<<<< HEAD
       {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}
     ] ++
       umbrella_apps() ++ enterprise_apps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
+=======
+      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
+      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.2"}
+    ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
+>>>>>>> fix: add influxdb udp api_v1 api_v2 connector
   end
 
   defp umbrella_apps() do

From fa54bf56128d44f7f64056b387dc69dd7741335d Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Sun, 31 Jul 2022 15:33:57 +0800
Subject: [PATCH 011/232] fix: influxdb connector api available

---
 .../src/emqx_ee_bridge_influxdb.erl           | 14 ++----
 .../src/emqx_ee_connector.erl                 | 14 +++---
 .../src/emqx_ee_connector_influxdb.erl        | 50 ++++++++++---------
 3 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 7004b70a1..dca55ba23 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -61,7 +61,11 @@ fields("config") ->
         {connector, field(connector)}
     ];
 fields("post") ->
-    [type_field(), name_field() | fields("config")];
+    [
+        {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})},
+        {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}
+        | fields("config")
+    ];
 fields("put") ->
     fields("config");
 fields("get") ->
@@ -89,11 +93,3 @@ desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for HStream using `", string:to_upper(Method), "` method."];
 desc(_) ->
     undefined.
-
-%% -------------------------------------------------------------------------------------------------
-%% internal
-type_field() ->
-    {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})}.
-
-name_field() ->
-    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
index 55ccc2c2c..9e8e7bac1 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
@@ -26,17 +26,16 @@ fields(connectors) ->
                 hoconsc:map(name, ref(emqx_ee_connector_hstream, config)),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
-    ];
-% ] ++ fields(influxdb);
+    ] ++ fields(influxdb);
 fields(influxdb) ->
     [
         {
-            influxdb,
-            mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, influxdb_udp)), #{
+            Protocol,
+            mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, Protocol)), #{
                 desc => <<"EMQX Enterprise Config">>
             })
         }
-        %  || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2]
+     || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2]
     ].
 
 connector_examples(Method) ->
@@ -52,4 +51,7 @@ connector_examples(Method) ->
     lists:foldl(Fun, #{}, schema_modules()).
 
 schema_modules() ->
-    [emqx_ee_connector_hstream, emqx_ee_connector_influxdb].
+    [
+        emqx_ee_connector_hstream,
+        emqx_ee_connector_influxdb
+    ].
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 1efc0b263..37915d2e8 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -11,8 +11,6 @@
 
 -behaviour(emqx_resource).
 
--define(PUT_FIELDS_FILTER, fun({Name, _}) -> not lists:member(Name, [type, name]) end).
-
 %% callbacks of behaviour emqx_resource
 -export([
     on_start/2,
@@ -50,23 +48,29 @@ on_get_status(_InstId, #{client := Client}) ->
 %% schema
 
 fields("put_udp") ->
-    lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_udp));
+    fields(influxdb_udp);
 fields("put_api_v1") ->
-    lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v1));
+    fields(influxdb_api_v1);
 fields("put_api_v2") ->
-    lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v2));
+    fields(influxdb_api_v2);
 fields("get_udp") ->
-    fields(influxdb_udp);
+    Key = influxdb_udp,
+    fields(Key) ++ type_name_field(Key);
 fields("get_api_v1") ->
-    fields(influxdb_api_v1);
+    Key = influxdb_api_v1,
+    fields(Key) ++ type_name_field(Key);
 fields("get_api_v2") ->
-    fields(influxdb_api_v2);
+    Key = influxdb_api_v2,
+    fields(Key) ++ type_name_field(Key);
 fields("post_udp") ->
-    fields(influxdb_udp);
+    Key = influxdb_udp,
+    fields(Key) ++ type_name_field(Key);
 fields("post_api_v1") ->
-    fields(influxdb_api_v1);
+    Key = influxdb_api_v1,
+    fields(Key) ++ type_name_field(Key);
 fields("post_api_v2") ->
-    fields(influxdb_api_v2);
+    Key = influxdb_api_v2,
+    fields(Key) ++ type_name_field(Key);
 fields(basic) ->
     [
         {host,
@@ -76,28 +80,29 @@ fields(basic) ->
             mk(enum([ns, us, ms, s, m, h]), #{
                 required => false, default => ms, desc => ?DESC("precision")
             })},
-        {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})},
-        {name, mk(binary(), #{required => true, desc => ?DESC("name")})}
+        {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}
     ];
 fields(influxdb_udp) ->
-    [
-        {type, mk(influxdb_udp, #{required => true, desc => ?DESC("type")})}
-    ] ++ fields(basic);
+    fields(basic);
 fields(influxdb_api_v1) ->
     [
-        {type, mk(influxdb_api_v1, #{required => true, desc => ?DESC("type")})},
         {database, mk(binary(), #{required => true, desc => ?DESC("database")})},
         {username, mk(binary(), #{required => true, desc => ?DESC("username")})},
         {password, mk(binary(), #{required => true, desc => ?DESC("password")})}
     ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic);
 fields(influxdb_api_v2) ->
     [
-        {type, mk(influxdb_api_v2, #{required => true, desc => ?DESC("type")})},
         {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})},
         {org, mk(binary(), #{required => true, desc => ?DESC("org")})},
         {token, mk(binary(), #{required => true, desc => ?DESC("token")})}
     ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic).
 
+type_name_field(Type) ->
+    [
+        {type, mk(Type, #{required => true, desc => ?DESC("type")})},
+        {name, mk(binary(), #{required => true, desc => ?DESC("name")})}
+    ].
+
 connector_examples(Method) ->
     [
         #{
@@ -123,9 +128,8 @@ connector_examples(Method) ->
 values(Protocol, get) ->
     values(Protocol, post);
 values(Protocol, post) ->
-    Type = list_to_atom(io_lib:format("influxdb_~p", [Protocol])),
-    ConnectorName = list_to_binary(io_lib:format("~p_connector", [Protocol])),
-    maps:merge(values(Protocol, put), #{type => Type, name => ConnectorName});
+    Type = list_to_atom("influxdb_" ++ atom_to_list(Protocol)),
+    maps:merge(values(Protocol, put), #{type => Type, name => <<"connector">>});
 values(udp, put) ->
     #{
         host => <<"127.0.0.1">>,
@@ -219,7 +223,7 @@ do_start_client(InstId, ClientConfig, Config = #{egress := #{payload := PayloadB
     end.
 
 client_config(
-    _InstId,
+    InstId,
     Config = #{
         host := Host,
         port := Port,
@@ -230,7 +234,7 @@ client_config(
         {host, Host},
         {port, Port},
         {pool_size, PoolSize},
-        {pool, atom_pname_todo},
+        {pool, binary_to_atom(InstId, utf8)},
         {precision, maps:get(precision, Config, ms)}
     ] ++ protocol_config(Config).
 

From 55b96845416a636fb831cbc28ca9d77ff04ed329 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Sun, 31 Jul 2022 20:37:13 +0800
Subject: [PATCH 012/232] fix: influxdb bridge api available

---
 apps/emqx_bridge/src/emqx_bridge_schema.erl   |  45 +++--
 .../i18n/emqx_ee_bridge_influxdb.conf         |  40 +++-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  37 +++-
 .../src/emqx_ee_bridge_hstream.erl            |  16 +-
 .../src/emqx_ee_bridge_influxdb.erl           | 135 +++++++++-----
 .../src/emqx_ee_connector.app.src             |   3 +-
 .../src/emqx_ee_connector_influxdb.erl        | 172 +++++++++++++++---
 7 files changed, 336 insertions(+), 112 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl
index 0b885db8c..46039390d 100644
--- a/apps/emqx_bridge/src/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl
@@ -36,24 +36,31 @@ post_request() ->
     http_schema("post").
 
 http_schema(Method) ->
-    Schemas = lists:flatmap(
-        fun(Type) ->
-            [
-                ref(schema_mod(Type), Method ++ "_ingress"),
-                ref(schema_mod(Type), Method ++ "_egress")
-            ]
-        end,
-        ?CONN_TYPES
-    ),
-    ExtSchemas = [ref(Module, Method) || Module <- schema_modules()],
-    hoconsc:union(Schemas ++ ExtSchemas).
+    Broker =
+        lists:flatmap(
+            fun(Type) ->
+                [
+                    ref(schema_mod(Type), Method ++ "_ingress"),
+                    ref(schema_mod(Type), Method ++ "_egress")
+                ]
+            end,
+            ?CONN_TYPES
+        ) ++ [ref(Module, Method) || Module <- [emqx_bridge_webhook_schema]],
+    EE = ee_schemas(Method),
+    hoconsc:union(Broker ++ EE).
 
 -if(?EMQX_RELEASE_EDITION == ee).
-schema_modules() ->
-    [emqx_bridge_webhook_schema] ++ emqx_ee_bridge:schema_modules().
+ee_schemas(Method) ->
+    emqx_ee_bridge:api_schemas(Method).
+
+ee_fields_bridges() ->
+    emqx_ee_bridge:fields(bridges).
 -else.
-schema_modules() ->
-    [emqx_bridge_webhook_schema].
+ee_schemas(_) ->
+    [].
+
+ee_fields_bridges() ->
+    [].
 -endif.
 
 common_bridge_fields(ConnectorRef) ->
@@ -158,14 +165,6 @@ fields("node_status") ->
         {"status", mk(status(), #{})}
     ].
 
--if(?EMQX_RELEASE_EDITION == ee).
-ee_fields_bridges() ->
-    emqx_ee_bridge:fields(bridges).
--else.
-ee_fields_bridges() ->
-    [].
--endif.
-
 desc(bridges) ->
     ?DESC("desc_bridges");
 desc("metrics") ->
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
index f421fb912..3930825e5 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -18,14 +18,44 @@ will be forwarded.
                 zh: "本地 Topic"
             }
     }
-    payload {
+    measurement {
         desc {
-            en: """The payload to be forwarded to the InfluxDB. Placeholders supported."""
-            zh: """要转发到 InfluxDB 的数据内容，支持占位符"""
+            en: """The measurement name to be forwarded to the InfluxDB. Placeholders supported."""
+            zh: """要转发到 InfluxDB 的 Measurement 名称，支持占位符"""
             }
         label {
-            en: "Payload"
-            zh: "消息内容"
+            en: "Measurement"
+            zh: "Measurement"
+        }
+    }
+    timestamp {
+        desc {
+            en: """The timestamp to be forwarded to the InfluxDB. Placeholders supported. Default is message timestamp"""
+            zh: """要转发到 InfluxDB 的时间戳，支持占位符。默认使用消息的时间戳"""
+            }
+        label {
+            en: "Timestamp"
+            zh: "Timestamp"
+        }
+    }
+    tags {
+        desc {
+            en: """The tags to be forwarded to the InfluxDB. Placeholders supported."""
+            zh: """要转发到 InfluxDB 的 Tags 数据内容，支持占位符"""
+            }
+        label {
+            en: "Tags"
+            zh: "Tags"
+        }
+    }
+    fields {
+        desc {
+            en: """The fields to be forwarded to the InfluxDB. Placeholders supported."""
+            zh: """要转发到 InfluxDB 的 fields 数据内容，支持占位符"""
+            }
+        label {
+            en: "Fields"
+            zh: "Fields"
         }
     }
     config_enable {
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index ff9661cc8..881609bfc 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -6,26 +6,43 @@
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -export([
-    schema_modules/0,
+    api_schemas/1,
     conn_bridge_examples/1,
     resource_type/1,
     fields/1
 ]).
 
+api_schemas(Method) ->
+    [
+        ref(emqx_ee_bridge_hstream, Method),
+        ref(emqx_ee_bridge_influxdb, Method ++ "_udp"),
+        ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"),
+        ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2")
+    ].
+
 schema_modules() ->
-    [emqx_ee_bridge_hstream, emqx_ee_bridge_influxdb].
+    [
+        emqx_ee_bridge_hstream,
+        emqx_ee_bridge_influxdb
+    ].
 
 conn_bridge_examples(Method) ->
+    MergeFun =
+        fun(Example, Examples) ->
+            maps:merge(Examples, Example)
+        end,
     Fun =
         fun(Module, Examples) ->
-            Example = erlang:apply(Module, conn_bridge_example, [Method]),
-            maps:merge(Examples, Example)
+            ConnectorExamples = erlang:apply(Module, conn_bridge_examples, [Method]),
+            lists:foldl(MergeFun, Examples, ConnectorExamples)
         end,
     lists:foldl(Fun, #{}, schema_modules()).
 
 resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8));
 resource_type(hstreamdb) -> emqx_ee_connector_hstream;
-resource_type(influxdb) -> emqx_ee_connector_influxdb.
+resource_type(influxdb_udp) -> emqx_ee_connector_influxdb;
+resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb;
+resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb.
 
 fields(bridges) ->
     [
@@ -33,10 +50,14 @@ fields(bridges) ->
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
                 #{desc => <<"EMQX Enterprise Config">>}
-            )},
-        {influxdb,
+            )}
+    ] ++ fields(influxdb);
+fields(influxdb) ->
+    [
+        {Protocol,
             mk(
-                hoconsc:map(name, ref(emqx_ee_bridge_influxdb, "config")),
+                hoconsc:map(name, ref(emqx_ee_bridge_influxdb, Protocol)),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
+     || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2]
     ].
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
index 73f7a20eb..200e695da 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
@@ -10,7 +10,7 @@
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -export([
-    conn_bridge_example/1
+    conn_bridge_examples/1
 ]).
 
 -export([
@@ -23,13 +23,15 @@
 %% -------------------------------------------------------------------------------------------------
 %% api
 
-conn_bridge_example(Method) ->
-    #{
-        <<"hstreamdb">> => #{
-            summary => <<"HStreamDB Bridge">>,
-            value => values(Method)
+conn_bridge_examples(Method) ->
+    [
+        #{
+            <<"hstreamdb">> => #{
+                summary => <<"HStreamDB Bridge">>,
+                value => values(Method)
+            }
         }
-    }.
+    ].
 
 values(get) ->
     maps:merge(values(post), ?METRICS_EXAMPLE);
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index dca55ba23..a19459208 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -10,7 +10,7 @@
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -export([
-    conn_bridge_example/1
+    conn_bridge_examples/1
 ]).
 
 -export([
@@ -23,28 +23,48 @@
 %% -------------------------------------------------------------------------------------------------
 %% api
 
-conn_bridge_example(Method) ->
-    #{
-        <<"influxdb">> => #{
-            summary => <<"InfluxDB Bridge">>,
-            value => values(Method)
+conn_bridge_examples(Method) ->
+    [
+        #{
+            <<"influxdb_udp">> => #{
+                summary => <<"InfluxDB UDP Bridge">>,
+                value => values("influxdb_udp", Method)
+            }
+        },
+        #{
+            <<"influxdb_api_v1">> => #{
+                summary => <<"InfluxDB HTTP API V1 Bridge">>,
+                value => values("influxdb_api_v1", Method)
+            }
+        },
+        #{
+            <<"influxdb_api_v2">> => #{
+                summary => <<"InfluxDB HTTP API V2 Bridge">>,
+                value => values("influxdb_api_v2", Method)
+            }
         }
-    }.
+    ].
 
-values(get) ->
-    maps:merge(values(post), ?METRICS_EXAMPLE);
-values(post) ->
+values(Protocol, get) ->
+    maps:merge(values(Protocol, post), ?METRICS_EXAMPLE);
+values(Protocol, post) ->
     #{
-        type => influxdb,
+        type => list_to_atom(Protocol),
         name => <<"demo">>,
-        connector => <<"influxdb:api_v2_connector">>,
+        connector => list_to_binary(Protocol ++ ":connector"),
         enable => true,
         direction => egress,
         local_topic => <<"local/topic/#">>,
-        payload => <<"${payload}">>
+        measurement => <<"${topic}">>,
+        tags => #{<<"clientid">> => <<"${clientid}">>},
+        fields => #{
+            <<"payload">> => <<"${payload}">>,
+            <<"int_value">> => [int, <<"${payload.int_key}">>],
+            <<"uint_value">> => [uint, <<"${payload.uint_key}">>]
+        }
     };
-values(put) ->
-    values(post).
+values(Protocol, put) ->
+    values(Protocol, post).
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
@@ -52,40 +72,69 @@ namespace() -> "bridge".
 
 roots() -> [].
 
-fields("config") ->
+fields(basic) ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
         {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
-        {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})},
-        {connector, field(connector)}
+        {measurement, mk(binary(), #{desc => ?DESC("measurement"), required => true})},
+        {timestamp,
+            mk(binary(), #{
+                desc => ?DESC("timestamp"), default => <<"${timestamp}">>, required => false
+            })},
+        {tags, mk(map(), #{desc => ?DESC("tags"), required => false})},
+        {fields, mk(map(), #{desc => ?DESC("fields"), required => true})}
     ];
-fields("post") ->
-    [
-        {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})},
-        {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}
-        | fields("config")
-    ];
-fields("put") ->
-    fields("config");
-fields("get") ->
-    emqx_bridge_schema:metrics_status_fields() ++ fields("post").
+fields("post_udp") ->
+    method_fileds(post, influxdb_udp);
+fields("post_api_v1") ->
+    method_fileds(post, influxdb_api_v1);
+fields("post_api_v2") ->
+    method_fileds(post, influxdb_api_v2);
+fields("put_udp") ->
+    method_fileds(put, influxdb_udp);
+fields("put_api_v1") ->
+    method_fileds(put, influxdb_api_v1);
+fields("put_api_v2") ->
+    method_fileds(put, influxdb_api_v2);
+fields("get_udp") ->
+    method_fileds(get, influxdb_udp);
+fields("get_api_v1") ->
+    method_fileds(get, influxdb_api_v1);
+fields("get_api_v2") ->
+    method_fileds(get, influxdb_api_v2);
+fields(Name) when
+    Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2
+->
+    fields(basic) ++ connector_field(Name).
 
-field(connector) ->
-    ConnectorConfigRef =
-        [
-            ref(emqx_ee_connector_influxdb, influxdb_udp),
-            ref(emqx_ee_connector_influxdb, influxdb_api_v1),
-            ref(emqx_ee_connector_influxdb, influxdb_api_v2)
-        ],
-    mk(
-        hoconsc:union([binary() | ConnectorConfigRef]),
-        #{
-            required => true,
-            example => <<"influxdb:demo">>,
-            desc => ?DESC("desc_connector")
-        }
-    ).
+method_fileds(post, ConnectorType) ->
+    fields(basic) ++ connector_field(ConnectorType) ++ type_name_field(ConnectorType);
+method_fileds(get, ConnectorType) ->
+    fields(basic) ++
+        emqx_bridge_schema:metrics_status_fields() ++
+        connector_field(ConnectorType) ++ type_name_field(ConnectorType);
+method_fileds(put, ConnectorType) ->
+    fields(basic) ++ connector_field(ConnectorType).
+
+connector_field(Type) ->
+    [
+        {connector,
+            mk(
+                hoconsc:union([binary(), ref(emqx_ee_connector_influxdb, Type)]),
+                #{
+                    required => true,
+                    example => list_to_binary(atom_to_list(Type) ++ ":connector"),
+                    desc => ?DESC(<<"desc_connector">>)
+                }
+            )}
+    ].
+
+type_name_field(Type) ->
+    [
+        {type, mk(Type, #{required => true, desc => ?DESC("desc_type")})},
+        {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}
+    ].
 
 desc("config") ->
     ?DESC("desc_config");
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 6d9fea3c9..675a934aa 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -4,7 +4,8 @@
     {applications, [
         kernel,
         stdlib,
-        hstreamdb_erl
+        hstreamdb_erl,
+        influxdb
     ]},
     {env, []},
     {modules, []},
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 37915d2e8..e343a64b0 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -33,8 +33,8 @@ on_start(InstId, Config) ->
 on_stop(_InstId, #{client := Client}) ->
     influxdb:stop_client(Client).
 
-on_query(_InstId, {send_message, _Data}, _AfterQuery, _State) ->
-    ok.
+on_query(InstId, {send_message, Data}, AfterQuery, State) ->
+    do_query(InstId, {send_message, Data}, AfterQuery, State).
 
 on_get_status(_InstId, #{client := Client}) ->
     case influxdb:is_alive(Client) of
@@ -184,18 +184,36 @@ start_client(InstId, Config) ->
             {error, R}
     end.
 
-do_start_client(InstId, ClientConfig, Config = #{egress := #{payload := PayloadBin}}) ->
+do_start_client(
+    InstId,
+    ClientConfig,
+    Config = #{
+        egress := #{
+            measurement := Measurement,
+            timestamp := Timestamp,
+            tags := Tags,
+            fields := Fields
+        }
+    }
+) ->
     case influxdb:start_client(ClientConfig) of
         {ok, Client} ->
             case influxdb:is_alive(Client) of
                 true ->
-                    Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin),
+                    State = #{
+                        client => Client,
+                        measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement),
+                        timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp),
+                        tags => to_tags_config(Tags),
+                        fields => to_fields_config(Fields)
+                    },
                     ?SLOG(info, #{
                         msg => "starting influxdb connector success",
                         connector => InstId,
-                        client => Client
+                        client => Client,
+                        state => State
                     }),
-                    #{client => Client, payload => Payload};
+                    {ok, State};
                 false ->
                     ?SLOG(error, #{
                         msg => "starting influxdb connector failed",
@@ -231,21 +249,15 @@ client_config(
     }
 ) ->
     [
-        {host, Host},
+        {host, binary_to_list(Host)},
         {port, Port},
         {pool_size, PoolSize},
         {pool, binary_to_atom(InstId, utf8)},
-        {precision, maps:get(precision, Config, ms)}
+        {precision, atom_to_binary(maps:get(precision, Config, ms), utf8)}
     ] ++ protocol_config(Config).
 
+%% api v2 config
 protocol_config(#{
-    protocol := udp
-}) ->
-    [
-        {protocol, udp}
-    ];
-protocol_config(#{
-    protocol := api_v1,
     username := Username,
     password := Password,
     database := DB,
@@ -254,13 +266,12 @@ protocol_config(#{
     [
         {protocol, http},
         {version, v1},
-        {username, Username},
-        {password, Password},
-        {database, DB},
-        {ssl, SSL}
+        {username, binary_to_list(Username)},
+        {password, binary_to_list(Password)},
+        {database, binary_to_list(DB)}
     ] ++ ssl_config(SSL);
+%% api v1 config
 protocol_config(#{
-    protocol := api_v2,
     bucket := Bucket,
     org := Org,
     token := Token,
@@ -269,11 +280,15 @@ protocol_config(#{
     [
         {protocol, http},
         {version, v2},
-        {bucket, Bucket},
-        {org, Org},
-        {token, Token},
-        {ssl, SSL}
-    ] ++ ssl_config(SSL).
+        {bucket, binary_to_list(Bucket)},
+        {org, binary_to_list(Org)},
+        {token, Token}
+    ] ++ ssl_config(SSL);
+%% udp config
+protocol_config(_) ->
+    [
+        {protocol, udp}
+    ].
 
 ssl_config(#{enable := false}) ->
     [
@@ -284,3 +299,110 @@ ssl_config(SSL = #{enable := true}) ->
         {https_enabled, true},
         {transport, ssl}
     ] ++ maps:to_list(maps:remove(enable, SSL)).
+
+%% -------------------------------------------------------------------------------------------------
+%% Query
+
+do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client}) ->
+    case data_to_point(Data, State) of
+        {ok, Point} ->
+            case influxdb:write(Client, [Point]) of
+                ok ->
+                    ?SLOG(debug, #{
+                        msg => "influxdb write point success",
+                        connector => InstId,
+                        point => Point
+                    }),
+                    emqx_resource:query_success(AfterQuery);
+                {error, Reason} ->
+                    ?SLOG(error, #{
+                        msg => "influxdb write point failed",
+                        connector => InstId,
+                        reason => Reason
+                    }),
+                    emqx_resource:query_failed(AfterQuery)
+            end;
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "influxdb trans point failed",
+                connector => InstId,
+                reason => Reason
+            }),
+            {error, Reason}
+    end.
+
+%% -------------------------------------------------------------------------------------------------
+%% Tags & Fields Config Trans
+
+to_tags_config(Tags) ->
+    maps:fold(fun to_maps_config/3, #{}, Tags).
+
+to_fields_config(Fields) ->
+    maps:fold(fun to_maps_config/3, #{}, Fields).
+
+to_maps_config(K, [IntType, V], Res) when IntType == <<"int">> orelse IntType == <<"uint">> ->
+    NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
+    NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)),
+    Res#{NK => {binary_to_atom(IntType, utf8), NV}};
+to_maps_config(K, V, Res) ->
+    NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
+    NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)),
+    Res#{NK => NV}.
+
+%% -------------------------------------------------------------------------------------------------
+%% Tags & Fields Data Trans
+data_to_point(
+    Data,
+    #{
+        measurement := Measurement,
+        timestamp := Timestamp,
+        tags := Tags,
+        fields := Fields
+    }
+) ->
+    TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
+    case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of
+        [TimestampInt] when is_integer(TimestampInt) ->
+            Point = #{
+                measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data),
+                timestamp => TimestampInt,
+                tags => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags),
+                fields => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields)
+            },
+            {ok, Point};
+        BadTimestamp ->
+            {error, {bad_timestamp, BadTimestamp}}
+    end.
+
+maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse IntType == uint ->
+    TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
+    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions),
+    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions),
+    case {NK, NV} of
+        {[undefined], _} ->
+            Res;
+        {_, [undefined]} ->
+            Res;
+        {_, [IntV]} when is_integer(IntV) ->
+            Res#{NK => {IntType, IntV}}
+    end;
+maps_config_to_data(K, V, {Data, Res}) ->
+    TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
+    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions),
+    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions),
+    case {NK, NV} of
+        {[undefined], _} ->
+            Res;
+        {_, [undefined]} ->
+            Res;
+        _ ->
+            Res#{bin(NK) => NV}
+    end.
+
+data_filter(undefined) -> undefined;
+data_filter(Int) when is_integer(Int) -> Int;
+data_filter(Number) when is_number(Number) -> Number;
+data_filter(Bool) when is_boolean(Bool) -> Bool;
+data_filter(Data) -> bin(Data).
+
+bin(Data) -> emqx_plugin_libs_rule:bin(Data).

From c1542e7a77bf7fcbf89f4299faa945d3e94e20a7 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Mon, 1 Aug 2022 10:03:16 +0800
Subject: [PATCH 013/232] fix: influxdb connector bad encode message

---
 .../src/emqx_ee_connector_influxdb.erl         | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index e343a64b0..c6528ffd0 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -363,11 +363,13 @@ data_to_point(
     TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
     case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of
         [TimestampInt] when is_integer(TimestampInt) ->
+            {_, EncodeTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags),
+            {_, EncodeFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields),
             Point = #{
                 measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data),
                 timestamp => TimestampInt,
-                tags => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags),
-                fields => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields)
+                tags => EncodeTags,
+                fields => EncodeFields
             },
             {ok, Point};
         BadTimestamp ->
@@ -380,11 +382,11 @@ maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse Int
     NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions),
     case {NK, NV} of
         {[undefined], _} ->
-            Res;
+            {Data, Res};
         {_, [undefined]} ->
-            Res;
+            {Data, Res};
         {_, [IntV]} when is_integer(IntV) ->
-            Res#{NK => {IntType, IntV}}
+            {Data, Res#{NK => {IntType, IntV}}}
     end;
 maps_config_to_data(K, V, {Data, Res}) ->
     TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
@@ -392,11 +394,11 @@ maps_config_to_data(K, V, {Data, Res}) ->
     NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions),
     case {NK, NV} of
         {[undefined], _} ->
-            Res;
+            {Data, Res};
         {_, [undefined]} ->
-            Res;
+            {Data, Res};
         _ ->
-            Res#{bin(NK) => NV}
+            {Data, Res#{bin(NK) => NV}}
     end.
 
 data_filter(undefined) -> undefined;

From 4663e354bf8cd394cbe00f43af8f28f4ee685196 Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Mon, 1 Aug 2022 10:31:33 +0800
Subject: [PATCH 014/232] fix: update influxdb sdk version to 1.1.3. Adapted
 master deps version

---
 lib-ee/emqx_ee_connector/rebar.config | 2 +-
 mix.exs                               | 9 ++-------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index 0a3f6866b..485b0120d 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, [
   {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
-  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.2"}}}
+  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.3"}}}
 ]}.
 
 {shell, [
diff --git a/mix.exs b/mix.exs
index 039172eb9..fa9593d07 100644
--- a/mix.exs
+++ b/mix.exs
@@ -89,15 +89,10 @@ defmodule EMQXUmbrella.MixProject do
        github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
       # in conflict by grpc and eetcd
       {:gpb, "4.11.2", override: true, runtime: false},
-<<<<<<< HEAD
-      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}
+      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
+      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.3", override: true}
     ] ++
       umbrella_apps() ++ enterprise_apps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
-=======
-      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
-      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.2"}
-    ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
->>>>>>> fix: add influxdb udp api_v1 api_v2 connector
   end
 
   defp umbrella_apps() do

From 78a44cfb874d0159073d5f60f43dc3dd3ad6af9e Mon Sep 17 00:00:00 2001
From: DDDHuang <44492639+DDDHuang@users.noreply.github.com>
Date: Mon, 1 Aug 2022 10:43:21 +0800
Subject: [PATCH 015/232] fix: override ehttpc & ecpool dep version

---
 mix.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index fa9593d07..77438d213 100644
--- a/mix.exs
+++ b/mix.exs
@@ -47,7 +47,7 @@ defmodule EMQXUmbrella.MixProject do
       {:lc, github: "emqx/lc", tag: "0.3.1"},
       {:redbug, "2.0.7"},
       {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
-      {:ehttpc, github: "emqx/ehttpc", tag: "0.3.0"},
+      {:ehttpc, github: "emqx/ehttpc", tag: "0.3.0", override: true},
       {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
       {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
@@ -56,7 +56,7 @@ defmodule EMQXUmbrella.MixProject do
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true},
-      {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
+      {:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true},
       {:replayq, "0.3.4", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
       {:emqtt, github: "emqx/emqtt", tag: "1.6.0", override: true},

From d778a71d3ba439b9442a8c2541aa61be7c271601 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 2 Aug 2022 16:45:47 +0800
Subject: [PATCH 016/232] chore: unifined lib-ee name style

---
 lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/.gitignore    | 0
 lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/README.md     | 0
 lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/rebar.config  | 0
 .../src/emqx_ee_conf.app.src}                               | 6 +++---
 .../src/emqx_ee_conf_schema.erl}                            | 0
 .../test/emqx_ee_conf_schema_SUITE.erl}                     | 0
 .../test/emqx_ee_conf_schema_tests.erl}                     | 0
 7 files changed, 3 insertions(+), 3 deletions(-)
 rename lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/.gitignore (100%)
 rename lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/README.md (100%)
 rename lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/rebar.config (100%)
 rename lib-ee/{emqx_enterprise_conf/src/emqx_enterprise_conf.app.src => emqx_ee_conf/src/emqx_ee_conf.app.src} (53%)
 rename lib-ee/{emqx_enterprise_conf/src/emqx_enterprise_conf_schema.erl => emqx_ee_conf/src/emqx_ee_conf_schema.erl} (100%)
 rename lib-ee/{emqx_enterprise_conf/test/emqx_enterprise_conf_schema_SUITE.erl => emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl} (100%)
 rename lib-ee/{emqx_enterprise_conf/test/emqx_enterprise_conf_schema_tests.erl => emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl} (100%)

diff --git a/lib-ee/emqx_enterprise_conf/.gitignore b/lib-ee/emqx_ee_conf/.gitignore
similarity index 100%
rename from lib-ee/emqx_enterprise_conf/.gitignore
rename to lib-ee/emqx_ee_conf/.gitignore
diff --git a/lib-ee/emqx_enterprise_conf/README.md b/lib-ee/emqx_ee_conf/README.md
similarity index 100%
rename from lib-ee/emqx_enterprise_conf/README.md
rename to lib-ee/emqx_ee_conf/README.md
diff --git a/lib-ee/emqx_enterprise_conf/rebar.config b/lib-ee/emqx_ee_conf/rebar.config
similarity index 100%
rename from lib-ee/emqx_enterprise_conf/rebar.config
rename to lib-ee/emqx_ee_conf/rebar.config
diff --git a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
similarity index 53%
rename from lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src
rename to lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
index a25ce1aae..771fdcb27 100644
--- a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src
+++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
@@ -1,6 +1,6 @@
-{application, emqx_enterprise_conf, [
-    {description, "EMQX Enterprise configuration schema"},
-    {vsn, "0.1.0"},
+{application, emqx_ee_conf, [
+    {description, "EMQX Enterprise Edition configuration schema"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl
similarity index 100%
rename from lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf_schema.erl
rename to lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl
diff --git a/lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_SUITE.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl
similarity index 100%
rename from lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_SUITE.erl
rename to lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl
diff --git a/lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_tests.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl
similarity index 100%
rename from lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_tests.erl
rename to lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl

From 10b19acd359e258eb13fbd7a7087cadce887fdd0 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 2 Aug 2022 16:58:21 +0800
Subject: [PATCH 017/232] chore: module and fun call rename

---
 build                                                |  2 +-
 git-blame-ignore-revs                                |  2 +-
 lib-ee/emqx_ee_conf/README.md                        |  2 +-
 lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src         |  2 +-
 lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl      |  2 +-
 .../emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl  | 12 ++++++------
 .../emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl  |  4 ++--
 lib-ee/emqx_license/test/emqx_license_SUITE.erl      |  6 +++---
 mix.exs                                              |  4 ++--
 rebar.config.erl                                     |  4 ++--
 10 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/build b/build
index 07c16b69e..f00cdab12 100755
--- a/build
+++ b/build
@@ -84,7 +84,7 @@ make_docs() {
     fi
     case $PROFILE in
         emqx-enterprise)
-            SCHEMA_MODULE='emqx_enterprise_conf_schema'
+            SCHEMA_MODULE='emqx_ee_conf_schema'
             ;;
         *)
             SCHEMA_MODULE='emqx_conf_schema'
diff --git a/git-blame-ignore-revs b/git-blame-ignore-revs
index b21b6a552..41c6e5e49 100644
--- a/git-blame-ignore-revs
+++ b/git-blame-ignore-revs
@@ -13,7 +13,7 @@ acb3544d4b112121b5d9414237d2af7860ccc2a3
 # reformat lib-ee/emqx_license
 4f396cceb84d79d5ef540e91c1a8420e8de74a56
 4e3fd9febd0df11f3fe5f221cd2c4362be57c886
-# reformat lib-ee/emqx_enterprise_conf
+# reformat lib-ee/emqx_ee_conf
 1aa82992616ad848539a533a5cd20ba6f9071e5a
 # reformat apps/emqx_gateway
 3f6d78dda03fd0d8e968a352e134f11a7f16bfe8
diff --git a/lib-ee/emqx_ee_conf/README.md b/lib-ee/emqx_ee_conf/README.md
index b5b28dfdb..701d285cc 100644
--- a/lib-ee/emqx_ee_conf/README.md
+++ b/lib-ee/emqx_ee_conf/README.md
@@ -1,3 +1,3 @@
-# emqx_enterprise_conf
+# emqx_ee_conf
 
 EMQX Enterprise configuration schema
diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
index 771fdcb27..324e7e308 100644
--- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
+++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_conf, [
     {description, "EMQX Enterprise Edition configuration schema"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.0"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl
index 60aeb1f81..38f6689c5 100644
--- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl
+++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl
@@ -2,7 +2,7 @@
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_enterprise_conf_schema).
+-module(emqx_ee_conf_schema).
 
 -behaviour(hocon_schema).
 
diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl
index 396faa4f5..0d6d4f061 100644
--- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl
+++ b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl
@@ -2,7 +2,7 @@
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_enterprise_conf_schema_SUITE).
+-module(emqx_ee_conf_schema_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -20,12 +20,12 @@ all() ->
 t_namespace(_Config) ->
     ?assertEqual(
         emqx_conf_schema:namespace(),
-        emqx_enterprise_conf_schema:namespace()
+        emqx_ee_conf_schema:namespace()
     ).
 
 t_roots(_Config) ->
     BaseRoots = emqx_conf_schema:roots(),
-    EnterpriseRoots = emqx_enterprise_conf_schema:roots(),
+    EnterpriseRoots = emqx_ee_conf_schema:roots(),
 
     ?assertEqual([], BaseRoots -- EnterpriseRoots),
 
@@ -42,12 +42,12 @@ t_roots(_Config) ->
 t_fields(_Config) ->
     ?assertEqual(
         emqx_conf_schema:fields("node"),
-        emqx_enterprise_conf_schema:fields("node")
+        emqx_ee_conf_schema:fields("node")
     ).
 
 t_translations(_Config) ->
-    [Root | _] = emqx_enterprise_conf_schema:translations(),
+    [Root | _] = emqx_ee_conf_schema:translations(),
     ?assertEqual(
         emqx_conf_schema:translation(Root),
-        emqx_enterprise_conf_schema:translation(Root)
+        emqx_ee_conf_schema:translation(Root)
     ).
diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl
index d7c4e35dd..b4bf0de3d 100644
--- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl
+++ b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl
@@ -2,7 +2,7 @@
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_enterprise_conf_schema_tests).
+-module(emqx_ee_conf_schema_tests).
 
 -include_lib("eunit/include/eunit.hrl").
 
@@ -22,7 +22,7 @@ doc_gen_test() ->
                 "priv",
                 "i18n.conf"
             ]),
-            _ = emqx_conf:dump_schema(Dir, emqx_enterprise_conf_schema, I18nFile),
+            _ = emqx_conf:dump_schema(Dir, emqx_ee_conf_schema, I18nFile),
             ok
         end
     }.
diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl
index a648595d2..ef7c86f8e 100644
--- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl
+++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl
@@ -104,10 +104,10 @@ setup_test(TestCase, Config) when
         [
             {apps, [emqx_conf, emqx_license]},
             {load_schema, false},
-            {schema_mod, emqx_enterprise_conf_schema},
+            {schema_mod, emqx_ee_conf_schema},
             {env_handler, fun
                 (emqx) ->
-                    emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema),
+                    emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema),
                     %% emqx_config:save_schema_mod_and_names(emqx_license_schema),
                     application:set_env(emqx, boot_modules, []),
                     application:set_env(
@@ -121,7 +121,7 @@ setup_test(TestCase, Config) when
                     ),
                     ok;
                 (emqx_conf) ->
-                    emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema),
+                    emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema),
                     %% emqx_config:save_schema_mod_and_names(emqx_license_schema),
                     application:set_env(
                         emqx,
diff --git a/mix.exs b/mix.exs
index 77438d213..5209de843 100644
--- a/mix.exs
+++ b/mix.exs
@@ -236,7 +236,7 @@ defmodule EMQXUmbrella.MixProject do
       if(edition_type == :enterprise,
         do: [
           emqx_license: :permanent,
-          emqx_enterprise_conf: :load,
+          emqx_ee_conf: :load,
           emqx_ee_connector: :permanent,
           emqx_ee_bridge: :permanent
         ],
@@ -603,7 +603,7 @@ defmodule EMQXUmbrella.MixProject do
     end
   end
 
-  defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_conf_schema
+  defp emqx_schema_mod(:enterprise), do: :emqx_ee_conf_schema
   defp emqx_schema_mod(:community), do: :emqx_conf_schema
 
   defp bcrypt_dep() do
diff --git a/rebar.config.erl b/rebar.config.erl
index 96754f7f8..0a5208b61 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -280,7 +280,7 @@ overlay_vars_edition(ce) ->
     ];
 overlay_vars_edition(ee) ->
     [
-        {emqx_schema_mod, emqx_enterprise_conf_schema},
+        {emqx_schema_mod, emqx_ee_conf_schema},
         {is_enterprise, "yes"}
     ].
 
@@ -378,7 +378,7 @@ is_app(Name) ->
 relx_apps_per_edition(ee) ->
     [
         emqx_license,
-        {emqx_enterprise_conf, load},
+        {emqx_ee_conf, load},
         emqx_ee_connector,
         emqx_ee_bridge
     ];

From 88fd7e14dc27dc7fd6dbde2fd32c879c2270b89a Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 28 Jul 2022 14:49:57 +0800
Subject: [PATCH 018/232] feat(bridge): add mysql sink

---
 .../i18n/emqx_ee_bridge_mysql.conf            |  75 +++++++++++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  10 +-
 .../src/emqx_ee_bridge_mysql.erl              | 104 ++++++++++++++++++
 .../i18n/emqx_ee_connector_mysql.conf         |  23 ++++
 .../src/emqx_ee_connector_mysql.erl           |  66 +++++++++++
 5 files changed, 277 insertions(+), 1 deletion(-)
 create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
 create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl

diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
new file mode 100644
index 000000000..9e3796487
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
@@ -0,0 +1,75 @@
+emqx_ee_bridge_mysql {
+    sql {
+        desc {
+            en: """SQL Template"""
+            zh: """SQL 模板"""
+            }
+        label {
+            en: "SQL Template"
+            zh: "SQL 模板"
+        }
+    }
+    config_enable {
+        desc {
+            en: """Enable or disable this bridge"""
+            zh: """启用/禁用桥接"""
+        }
+        label {
+            en: "Enable Or Disable Bridge"
+            zh: "启用/禁用桥接"
+        }
+        }
+    config_direction {
+        desc {
+            en: """The direction of this bridge, MUST be 'egress'"""
+            zh: """桥接的方向， 必须是 egress"""
+        }
+        label {
+            en: "Bridge Direction"
+            zh: "桥接方向"
+        }
+    }
+
+    desc_config {
+        desc {
+            en: """Configuration for an HStreamDB bridge."""
+            zh: """HStreamDB 桥接配置"""
+        }
+        label: {
+            en: "HStreamDB Bridge Configuration"
+            zh: "HStreamDB 桥接配置"
+        }
+    }
+
+    desc_type {
+        desc {
+            en: """The Bridge Type"""
+            zh: """Bridge 类型"""
+        }
+        label {
+            en: "Bridge Type"
+            zh: "桥接类型"
+        }
+    }
+
+    desc_name {
+        desc {
+            en: """Bridge name, used as a human-readable description of the bridge."""
+            zh: """桥接名字，可读描述"""
+        }
+        label {
+            en: "Bridge Name"
+            zh: "桥接名字"
+        }
+    }
+    desc_connector {
+        desc {
+                en: """Generic configuration for the connector."""
+                zh: """连接器的通用配置。"""
+        }
+        label: {
+                en: "Connector Generic Configuration"
+                zh: "连接器通用配置。"
+        }
+    }
+}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 881609bfc..5132ee249 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -15,6 +15,7 @@
 api_schemas(Method) ->
     [
         ref(emqx_ee_bridge_hstream, Method),
+        ref(emqx_ee_bridge_mysql, Method),
         ref(emqx_ee_bridge_influxdb, Method ++ "_udp"),
         ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"),
         ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2")
@@ -23,7 +24,8 @@ api_schemas(Method) ->
 schema_modules() ->
     [
         emqx_ee_bridge_hstream,
-        emqx_ee_bridge_influxdb
+        emqx_ee_bridge_influxdb,
+        emqx_ee_bridge_mysql
     ].
 
 conn_bridge_examples(Method) ->
@@ -40,6 +42,7 @@ conn_bridge_examples(Method) ->
 
 resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8));
 resource_type(hstreamdb) -> emqx_ee_connector_hstream;
+resource_type(mysql) -> emqx_ee_connector_mysql;
 resource_type(influxdb_udp) -> emqx_ee_connector_influxdb;
 resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb;
 resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb.
@@ -50,6 +53,11 @@ fields(bridges) ->
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
                 #{desc => <<"EMQX Enterprise Config">>}
+            )},
+        {mysql,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_bridge_mysql, "config")),
+                #{desc => <<"EMQX Enterprise Config">>}
             )}
     ] ++ fields(influxdb);
 fields(influxdb) ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
new file mode 100644
index 000000000..7d586d7eb
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -0,0 +1,104 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_mysql).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+-include("emqx_ee_bridge.hrl").
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-export([
+    conn_bridge_examples/1
+]).
+
+-export([
+    namespace/0,
+    roots/0,
+    fields/1,
+    desc/1
+]).
+
+-define(DEFAULT_SQL, <<
+    "insert into t_mqtt_msg(msgid, topic, qos, payload, arrived) "
+    "values (${id}, ${topic}, ${qos}, ${payload}, FROM_UNIXTIME(${timestamp}/1000))"
+>>).
+
+%% -------------------------------------------------------------------------------------------------
+%% api
+
+conn_bridge_examples(Method) ->
+    [
+        #{
+            <<"mysql">> => #{
+                summary => <<"MySQL Bridge">>,
+                value => values(Method)
+            }
+        }
+    ].
+
+values(get) ->
+    maps:merge(values(post), ?METRICS_EXAMPLE);
+values(post) ->
+    #{
+        type => mysql,
+        name => <<"mysql">>,
+        connector => #{
+            server => <<"127.0.0.1:3306">>,
+            database => <<"test">>,
+            pool_size => 8,
+            username => <<"root">>,
+            password => <<"public">>,
+            auto_reconnect => true
+        },
+        enable => true,
+        direction => egress,
+        sql => ?DEFAULT_SQL
+    };
+values(put) ->
+    values(post).
+
+%% -------------------------------------------------------------------------------------------------
+%% Hocon Schema Definitions
+namespace() -> "bridge".
+
+roots() -> [].
+
+fields("config") ->
+    [
+        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
+        {sql, mk(binary(), #{default => ?DEFAULT_SQL, desc => ?DESC("sql")})},
+        {connector, field(connector)}
+    ];
+fields("post") ->
+    [type_field(), name_field() | fields("config")];
+fields("put") ->
+    fields("config");
+fields("get") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields("post").
+
+field(connector) ->
+    mk(
+        ref(emqx_ee_connector_mysql, config),
+        #{
+            required => true,
+            desc => ?DESC("desc_connector")
+        }
+    ).
+
+desc("config") ->
+    ?DESC("desc_config");
+desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
+    ["Configuration for MySQL using `", string:to_upper(Method), "` method."];
+desc(_) ->
+    undefined.
+
+%% -------------------------------------------------------------------------------------------------
+%% internal
+type_field() ->
+    {type, mk(enum([mysql]), #{required => true, desc => ?DESC("desc_type")})}.
+
+name_field() ->
+    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf
new file mode 100644
index 000000000..5d7c4848a
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf
@@ -0,0 +1,23 @@
+emqx_ee_connector_mysql {
+    type {
+        desc {
+                en: "The Connector Type."
+                zh: "连接器类型。"
+        }
+        label: {
+                en: "Connector Type"
+                zh: "连接器类型"
+            }
+    }
+
+    name {
+        desc {
+                en: "Connector name, used as a human-readable description of the connector."
+                zh: "连接器名称，人类可读的连接器描述。"
+        }
+        label: {
+                en: "Connector Name"
+                zh: "连接器名称"
+            }
+    }
+}
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl
new file mode 100644
index 000000000..c4efff91c
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl
@@ -0,0 +1,66 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_connector_mysql).
+
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+-import(hoconsc, [mk/2, enum/1]).
+
+-behaviour(emqx_resource).
+
+%% callbacks of behaviour emqx_resource
+-export([
+    on_start/2,
+    on_stop/2,
+    on_query/4,
+    on_get_status/2
+]).
+
+-export([
+    roots/0,
+    fields/1
+]).
+
+-define(SEND_MSG_KEY, send_message).
+
+%% -------------------------------------------------------------------------------------------------
+%% resource callback
+
+on_start(InstId, #{egress := #{sql := SQL}} = Config) ->
+    {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'),
+    {ok, State} = emqx_connector_mysql:on_start(InstId, Config#{
+        prepare_statement => #{?SEND_MSG_KEY => PrepareSQL}
+    }),
+    {ok, State#{'ParamsTokens' => ParamsTokens}}.
+
+on_stop(InstId, State) ->
+    emqx_connector_mysql:on_stop(InstId, State).
+
+on_query(
+    InstId,
+    {?SEND_MSG_KEY, Msg},
+    AfterQuery,
+    #{'ParamsTokens' := ParamsTokens} = State
+) ->
+    Data = emqx_plugin_libs_rule:proc_sql(ParamsTokens, Msg),
+    emqx_connector_mysql:on_query(
+        InstId, {prepared_query, ?SEND_MSG_KEY, Data}, AfterQuery, State
+    ).
+
+on_get_status(InstId, State) ->
+    emqx_connector_mysql:on_get_status(InstId, State).
+
+%% -------------------------------------------------------------------------------------------------
+%% schema
+
+roots() ->
+    fields(config).
+
+fields(config) ->
+    emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields().
+
+%% -------------------------------------------------------------------------------------------------
+%% internal functions

From aa6077bdeaee1e08f89329b39a2f177adda69db7 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 4 Aug 2022 14:52:16 +0800
Subject: [PATCH 019/232] fix(bridge): remove emqx_ee_connector_mysql

---
 .../src/emqx_connector_mysql.erl              | 62 +++++++++++++----
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  2 +-
 .../src/emqx_ee_bridge_mysql.erl              | 42 +++++++-----
 .../i18n/emqx_ee_connector_mysql.conf         | 23 -------
 .../src/emqx_ee_connector_mysql.erl           | 66 -------------------
 5 files changed, 78 insertions(+), 117 deletions(-)
 delete mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf
 delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index d6963d04e..409da4060 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -44,6 +44,16 @@
     default_port => ?MYSQL_DEFAULT_PORT
 }).
 
+-type prepares() :: #{atom() => binary()}.
+-type params_tokens() :: #{atom() => list()}.
+-type state() ::
+    #{
+        poolname := atom(),
+        prepare_statement := prepares(),
+        auto_reconnect := boolean(),
+        params_tokens := params_tokens()
+    }.
+
 %%=====================================================================
 %% Hocon schema
 roots() ->
@@ -63,6 +73,7 @@ server(desc) -> ?DESC("server");
 server(_) -> undefined.
 
 %% ===================================================================
+-spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}.
 on_start(
     InstId,
     #{
@@ -97,8 +108,8 @@ on_start(
         {pool_size, PoolSize}
     ],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
-    Prepares = maps:get(prepare_statement, Config, #{}),
-    State = #{poolname => PoolName, prepare_statement => Prepares, auto_reconnect => AutoReconn},
+    Prepares = parse_prepare_sql(maps:get(prepare_statement, Config, #{})),
+    State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
         ok -> {ok, init_prepare(State)};
         {error, Reason} -> {error, Reason}
@@ -111,13 +122,13 @@ on_stop(InstId, #{poolname := PoolName}) ->
     }),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
-on_query(InstId, {Type, SQLOrKey}, AfterQuery, State) ->
-    on_query(InstId, {Type, SQLOrKey, [], default_timeout}, AfterQuery, State);
-on_query(InstId, {Type, SQLOrKey, Params}, AfterQuery, State) ->
-    on_query(InstId, {Type, SQLOrKey, Params, default_timeout}, AfterQuery, State);
+on_query(InstId, {TypeOrKey, SQLOrKey}, AfterQuery, State) ->
+    on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, AfterQuery, State);
+on_query(InstId, {TypeOrKey, SQLOrKey, Params}, AfterQuery, State) ->
+    on_query(InstId, {TypeOrKey, SQLOrKey, Params, default_timeout}, AfterQuery, State);
 on_query(
     InstId,
-    {Type, SQLOrKey, Params, Timeout},
+    {TypeOrKey, SQLOrKey, Params, Timeout},
     AfterQuery,
     #{poolname := PoolName, prepare_statement := Prepares} = State
 ) ->
@@ -125,8 +136,9 @@ on_query(
     ?TRACE("QUERY", "mysql_connector_received", LogMeta),
     Worker = ecpool:get_client(PoolName),
     {ok, Conn} = ecpool_worker:client(Worker),
-    MySqlFunction = mysql_function(Type),
-    Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey, Params, Timeout]),
+    MySqlFunction = mysql_function(TypeOrKey),
+    {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State),
+    Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey2, Data, Timeout]),
     case Result of
         {error, disconnected} ->
             ?SLOG(
@@ -145,7 +157,7 @@ on_query(
             case prepare_sql(Prepares, PoolName) of
                 ok ->
                     %% not return result, next loop will try again
-                    on_query(InstId, {Type, SQLOrKey, Params, Timeout}, AfterQuery, State);
+                    on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, AfterQuery, State);
                 {error, Reason} ->
                     ?SLOG(
                         error,
@@ -166,8 +178,13 @@ on_query(
             Result
     end.
 
-mysql_function(sql) -> query;
-mysql_function(prepared_query) -> execute.
+mysql_function(sql) ->
+    query;
+mysql_function(prepared_query) ->
+    execute;
+%% for bridge
+mysql_function(_) ->
+    mysql_function(prepared_query).
 
 on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) ->
     case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of
@@ -287,3 +304,24 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) ->
 
 unprepare_sql_to_conn(Conn, PrepareSqlKey) ->
     mysql:unprepare(Conn, PrepareSqlKey).
+
+parse_prepare_sql(SQL) ->
+    parse_prepare_sql(maps:to_list(SQL), #{}, #{}).
+
+parse_prepare_sql([{Key, H} | T], SQL, Tokens) ->
+    {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H),
+    parse_prepare_sql(T, SQL#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens});
+parse_prepare_sql([], SQL, Tokens) ->
+    #{prepare_statement => SQL, params_tokens => Tokens}.
+
+proc_sql_params(query, SQLOrKey, Params, _State) ->
+    {SQLOrKey, Params};
+proc_sql_params(prepared_query, SQLOrKey, Params, _State) ->
+    {SQLOrKey, Params};
+proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) ->
+    case maps:get(TypeOrKey, ParamsTokens, undefined) of
+        undefined ->
+            {SQLOrData, Params};
+        Tokens ->
+            {TypeOrKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
+    end.
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 5132ee249..a690b50d0 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -42,7 +42,7 @@ conn_bridge_examples(Method) ->
 
 resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8));
 resource_type(hstreamdb) -> emqx_ee_connector_hstream;
-resource_type(mysql) -> emqx_ee_connector_mysql;
+resource_type(mysql) -> emqx_connector_mysql;
 resource_type(influxdb_udp) -> emqx_ee_connector_influxdb;
 resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb;
 resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb.
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index 7d586d7eb..bf38da2f8 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -50,11 +50,11 @@ values(post) ->
             pool_size => 8,
             username => <<"root">>,
             password => <<"public">>,
-            auto_reconnect => true
+            auto_reconnect => true,
+            prepare_statement => #{send_message => ?DEFAULT_SQL}
         },
         enable => true,
-        direction => egress,
-        sql => ?DEFAULT_SQL
+        direction => egress
     };
 values(put) ->
     values(post).
@@ -69,27 +69,29 @@ fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
-        {sql, mk(binary(), #{default => ?DEFAULT_SQL, desc => ?DESC("sql")})},
-        {connector, field(connector)}
+        {connector,
+            mk(
+                ref(?MODULE, connector),
+                #{
+                    required => true,
+                    desc => ?DESC("desc_connector")
+                }
+            )}
     ];
 fields("post") ->
     [type_field(), name_field() | fields("config")];
 fields("put") ->
     fields("config");
 fields("get") ->
-    emqx_bridge_schema:metrics_status_fields() ++ fields("post").
-
-field(connector) ->
-    mk(
-        ref(emqx_ee_connector_mysql, config),
-        #{
-            required => true,
-            desc => ?DESC("desc_connector")
-        }
-    ).
+    emqx_bridge_schema:metrics_status_fields() ++ fields("post");
+fields(connector) ->
+    (emqx_connector_mysql:fields(config) --
+        emqx_connector_schema_lib:prepare_statement_fields()) ++ prepare_statement_fields().
 
 desc("config") ->
     ?DESC("desc_config");
+desc(connector) ->
+    ?DESC("desc_connector");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for MySQL using `", string:to_upper(Method), "` method."];
 desc(_) ->
@@ -102,3 +104,13 @@ type_field() ->
 
 name_field() ->
     {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
+
+prepare_statement_fields() ->
+    [
+        {prepare_statement,
+            mk(map(), #{
+                desc => ?DESC(emqx_connector_schema_lib, prepare_statement),
+                default => #{<<"send_message">> => ?DEFAULT_SQL},
+                example => #{<<"send_message">> => ?DEFAULT_SQL}
+            })}
+    ].
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf
deleted file mode 100644
index 5d7c4848a..000000000
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-emqx_ee_connector_mysql {
-    type {
-        desc {
-                en: "The Connector Type."
-                zh: "连接器类型。"
-        }
-        label: {
-                en: "Connector Type"
-                zh: "连接器类型"
-            }
-    }
-
-    name {
-        desc {
-                en: "Connector name, used as a human-readable description of the connector."
-                zh: "连接器名称，人类可读的连接器描述。"
-        }
-        label: {
-                en: "Connector Name"
-                zh: "连接器名称"
-            }
-    }
-}
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl
deleted file mode 100644
index c4efff91c..000000000
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl
+++ /dev/null
@@ -1,66 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
--module(emqx_ee_connector_mysql).
-
--include_lib("hocon/include/hoconsc.hrl").
--include_lib("typerefl/include/types.hrl").
--include_lib("emqx/include/logger.hrl").
-
--import(hoconsc, [mk/2, enum/1]).
-
--behaviour(emqx_resource).
-
-%% callbacks of behaviour emqx_resource
--export([
-    on_start/2,
-    on_stop/2,
-    on_query/4,
-    on_get_status/2
-]).
-
--export([
-    roots/0,
-    fields/1
-]).
-
--define(SEND_MSG_KEY, send_message).
-
-%% -------------------------------------------------------------------------------------------------
-%% resource callback
-
-on_start(InstId, #{egress := #{sql := SQL}} = Config) ->
-    {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'),
-    {ok, State} = emqx_connector_mysql:on_start(InstId, Config#{
-        prepare_statement => #{?SEND_MSG_KEY => PrepareSQL}
-    }),
-    {ok, State#{'ParamsTokens' => ParamsTokens}}.
-
-on_stop(InstId, State) ->
-    emqx_connector_mysql:on_stop(InstId, State).
-
-on_query(
-    InstId,
-    {?SEND_MSG_KEY, Msg},
-    AfterQuery,
-    #{'ParamsTokens' := ParamsTokens} = State
-) ->
-    Data = emqx_plugin_libs_rule:proc_sql(ParamsTokens, Msg),
-    emqx_connector_mysql:on_query(
-        InstId, {prepared_query, ?SEND_MSG_KEY, Data}, AfterQuery, State
-    ).
-
-on_get_status(InstId, State) ->
-    emqx_connector_mysql:on_get_status(InstId, State).
-
-%% -------------------------------------------------------------------------------------------------
-%% schema
-
-roots() ->
-    fields(config).
-
-fields(config) ->
-    emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields().
-
-%% -------------------------------------------------------------------------------------------------
-%% internal functions

From 4d0516c6e9bd3d2e1ba156e78ac096c58a2af581 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 4 Aug 2022 10:10:38 +0800
Subject: [PATCH 020/232] chore: use HStreamDB for module name

---
 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf    | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl              | 8 ++++----
 ...ee_bridge_hstream.erl => emqx_ee_bridge_hstreamdb.erl} | 4 ++--
 ...ge_hstream_SUITE.erl => ee_bridge_hstreamdb_SUITE.erl} | 2 +-
 .../emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf | 2 +-
 lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl        | 6 +++---
 ...nector_hstream.erl => emqx_ee_connector_hstreamdb.erl} | 2 +-
 ...hstream_SUITE.erl => ee_connector_hstreamdb_SUITE.erl} | 2 +-
 8 files changed, 14 insertions(+), 14 deletions(-)
 rename lib-ee/emqx_ee_bridge/src/{emqx_ee_bridge_hstream.erl => emqx_ee_bridge_hstreamdb.erl} (95%)
 rename lib-ee/emqx_ee_bridge/test/{ee_bridge_hstream_SUITE.erl => ee_bridge_hstreamdb_SUITE.erl} (92%)
 rename lib-ee/emqx_ee_connector/src/{emqx_ee_connector_hstream.erl => emqx_ee_connector_hstreamdb.erl} (99%)
 rename lib-ee/emqx_ee_connector/test/{ee_connector_hstream_SUITE.erl => ee_connector_hstreamdb_SUITE.erl} (91%)

diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
index 2e4397a04..dd3346579 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
@@ -1,4 +1,4 @@
-emqx_ee_bridge_hstream {
+emqx_ee_bridge_hstreamdb {
     local_topic {
         desc {
             en: """
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index a690b50d0..ebc06d211 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -14,8 +14,8 @@
 
 api_schemas(Method) ->
     [
-        ref(emqx_ee_bridge_hstream, Method),
         ref(emqx_ee_bridge_mysql, Method),
+        ref(emqx_ee_bridge_hstreamdb, Method),
         ref(emqx_ee_bridge_influxdb, Method ++ "_udp"),
         ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"),
         ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2")
@@ -23,7 +23,7 @@ api_schemas(Method) ->
 
 schema_modules() ->
     [
-        emqx_ee_bridge_hstream,
+        emqx_ee_bridge_hstreamdb,
         emqx_ee_bridge_influxdb,
         emqx_ee_bridge_mysql
     ].
@@ -41,7 +41,7 @@ conn_bridge_examples(Method) ->
     lists:foldl(Fun, #{}, schema_modules()).
 
 resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8));
-resource_type(hstreamdb) -> emqx_ee_connector_hstream;
+resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb;
 resource_type(mysql) -> emqx_connector_mysql;
 resource_type(influxdb_udp) -> emqx_ee_connector_influxdb;
 resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb;
@@ -51,7 +51,7 @@ fields(bridges) ->
     [
         {hstreamdb,
             mk(
-                hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")),
+                hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")),
                 #{desc => <<"EMQX Enterprise Config">>}
             )},
         {mysql,
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
similarity index 95%
rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
rename to lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
index 200e695da..79a50f35f 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_bridge_hstream).
+-module(emqx_ee_bridge_hstreamdb).
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -71,7 +71,7 @@ fields("get") ->
 
 field(connector) ->
     mk(
-        hoconsc:union([binary(), ref(emqx_ee_connector_hstream, config)]),
+        hoconsc:union([binary(), ref(emqx_ee_connector_hstreamdb, config)]),
         #{
             required => true,
             example => <<"hstreamdb:demo">>,
diff --git a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl
similarity index 92%
rename from lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl
rename to lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl
index d03c13cac..429323ad7 100644
--- a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl
@@ -2,7 +2,7 @@
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(ee_bridge_hstream_SUITE).
+-module(ee_bridge_hstreamdb_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
index 080076065..dd9659aba 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
@@ -1,5 +1,5 @@
 
-emqx_ee_connector_hstream {
+emqx_ee_connector_hstreamdb {
     type {
         desc {
                 en: "The Connector Type."
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
index 9e8e7bac1..0971e1a06 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
@@ -13,7 +13,7 @@
 
 api_schemas(Method) ->
     [
-        ref(emqx_ee_connector_hstream, Method),
+        ref(emqx_ee_connector_hstreamdb, Method),
         ref(emqx_ee_connector_influxdb, Method ++ "_udp"),
         ref(emqx_ee_connector_influxdb, Method ++ "_api_v1"),
         ref(emqx_ee_connector_influxdb, Method ++ "_api_v2")
@@ -23,7 +23,7 @@ fields(connectors) ->
     [
         {hstreamdb,
             mk(
-                hoconsc:map(name, ref(emqx_ee_connector_hstream, config)),
+                hoconsc:map(name, ref(emqx_ee_connector_hstreamdb, config)),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
     ] ++ fields(influxdb);
@@ -52,6 +52,6 @@ connector_examples(Method) ->
 
 schema_modules() ->
     [
-        emqx_ee_connector_hstream,
+        emqx_ee_connector_hstreamdb,
         emqx_ee_connector_influxdb
     ].
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
similarity index 99%
rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
rename to lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
index 7a8ce3fe9..56e6e536c 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_connector_hstream).
+-module(emqx_ee_connector_hstreamdb).
 
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("typerefl/include/types.hrl").
diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl
similarity index 91%
rename from lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl
rename to lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl
index cebe77c6a..4de456b2b 100644
--- a/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl
+++ b/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl
@@ -2,7 +2,7 @@
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(ee_connector_hstream_SUITE).
+-module(ee_connector_hstreamdb_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).

From 2e336fbc018e44e4433d85893fa0f0bf0f49d457 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 4 Aug 2022 10:17:09 +0800
Subject: [PATCH 021/232] fix: bridge and connector namespace contains DB name

Contains Detailed DB name for identifier hocon schema refs.
---
 apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl          | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl       | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl        | 2 +-
 lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl | 2 ++
 lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl  | 2 ++
 5 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl
index f11247d68..d833e6ca8 100644
--- a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl
@@ -9,7 +9,7 @@
 
 %%======================================================================================
 %% Hocon Schema Definitions
-namespace() -> "bridge".
+namespace() -> "bridge_webhook".
 
 roots() -> [].
 
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
index 79a50f35f..3b5183150 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
@@ -50,7 +50,7 @@ values(put) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
-namespace() -> "bridge".
+namespace() -> "bridge_hstreamdb".
 
 roots() -> [].
 
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index a19459208..a55d9d47a 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -68,7 +68,7 @@ values(Protocol, put) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
-namespace() -> "bridge".
+namespace() -> "bridge_influxdb".
 
 roots() -> [].
 
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
index 56e6e536c..d0bbed136 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
@@ -24,6 +24,7 @@
 ]).
 
 -export([
+    namespace/0,
     roots/0,
     fields/1,
     connector_examples/1
@@ -75,6 +76,7 @@ on_flush_result({{flush, _Stream, _Records}, {error, _Reason}}) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% schema
+namespace() -> connector_hstreamdb.
 
 roots() ->
     fields(config).
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index c6528ffd0..8ba8478b3 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -20,6 +20,7 @@
 ]).
 
 -export([
+    namespace/0,
     fields/1,
     connector_examples/1
 ]).
@@ -46,6 +47,7 @@ on_get_status(_InstId, #{client := Client}) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% schema
+namespace() -> connector_influxdb.
 
 fields("put_udp") ->
     fields(influxdb_udp);

From 33a604dcaa1d0b3d4e50ee4a6b8def72ecc399e6 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 4 Aug 2022 13:40:00 +0800
Subject: [PATCH 022/232] fix: refine influxdb connector api type name

---
 .../src/emqx_ee_connector.erl                 |  6 +--
 .../src/emqx_ee_connector_influxdb.erl        | 42 +++++++++----------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
index 0971e1a06..6846ea740 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
@@ -14,9 +14,9 @@
 api_schemas(Method) ->
     [
         ref(emqx_ee_connector_hstreamdb, Method),
-        ref(emqx_ee_connector_influxdb, Method ++ "_udp"),
-        ref(emqx_ee_connector_influxdb, Method ++ "_api_v1"),
-        ref(emqx_ee_connector_influxdb, Method ++ "_api_v2")
+        ref(emqx_ee_connector_influxdb, "udp_" ++ Method),
+        ref(emqx_ee_connector_influxdb, "api_v1_" ++ Method),
+        ref(emqx_ee_connector_influxdb, "api_v2_" ++ Method)
     ].
 
 fields(connectors) ->
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 8ba8478b3..c9971607e 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -49,30 +49,30 @@ on_get_status(_InstId, #{client := Client}) ->
 %% schema
 namespace() -> connector_influxdb.
 
-fields("put_udp") ->
+fields("udp_get") ->
+    Key = influxdb_udp,
+    fields(Key) ++ type_name_field(Key);
+fields("udp_post") ->
+    Key = influxdb_udp,
+    fields(Key) ++ type_name_field(Key);
+fields("udp_put") ->
     fields(influxdb_udp);
-fields("put_api_v1") ->
+fields("api_v1_get") ->
+    Key = influxdb_api_v1,
+    fields(Key) ++ type_name_field(Key);
+fields("api_v1_post") ->
+    Key = influxdb_api_v1,
+    fields(Key) ++ type_name_field(Key);
+fields("api_v1_put") ->
     fields(influxdb_api_v1);
-fields("put_api_v2") ->
+fields("api_v2_get") ->
+    Key = influxdb_api_v2,
+    fields(Key) ++ type_name_field(Key);
+fields("api_v2_post") ->
+    Key = influxdb_api_v2,
+    fields(Key) ++ type_name_field(Key);
+fields("api_v2_put") ->
     fields(influxdb_api_v2);
-fields("get_udp") ->
-    Key = influxdb_udp,
-    fields(Key) ++ type_name_field(Key);
-fields("get_api_v1") ->
-    Key = influxdb_api_v1,
-    fields(Key) ++ type_name_field(Key);
-fields("get_api_v2") ->
-    Key = influxdb_api_v2,
-    fields(Key) ++ type_name_field(Key);
-fields("post_udp") ->
-    Key = influxdb_udp,
-    fields(Key) ++ type_name_field(Key);
-fields("post_api_v1") ->
-    Key = influxdb_api_v1,
-    fields(Key) ++ type_name_field(Key);
-fields("post_api_v2") ->
-    Key = influxdb_api_v2,
-    fields(Key) ++ type_name_field(Key);
 fields(basic) ->
     [
         {host,

From 06f246a8960fe8c2988604552e7f436da87a18f0 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 8 Aug 2022 10:38:52 +0800
Subject: [PATCH 023/232] fix: prevent unexcepted square brackets in influxdb
 line protocol

---
 .../src/emqx_ee_connector_influxdb.erl           | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index c9971607e..f0e5cd333 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -379,9 +379,10 @@ data_to_point(
     end.
 
 maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse IntType == uint ->
-    TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
-    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions),
-    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions),
+    KTransOptions = #{return => full_binary},
+    VTransOptions = #{return => rawlist, var_trans => fun data_filter/1},
+    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions),
+    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions),
     case {NK, NV} of
         {[undefined], _} ->
             {Data, Res};
@@ -391,16 +392,17 @@ maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse Int
             {Data, Res#{NK => {IntType, IntV}}}
     end;
 maps_config_to_data(K, V, {Data, Res}) ->
-    TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
-    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions),
-    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions),
+    KTransOptions = #{return => full_binary},
+    VTransOptions = #{return => rawlist, var_trans => fun data_filter/1},
+    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions),
+    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions),
     case {NK, NV} of
         {[undefined], _} ->
             {Data, Res};
         {_, [undefined]} ->
             {Data, Res};
         _ ->
-            {Data, Res#{bin(NK) => NV}}
+            {Data, Res#{NK => NV}}
     end.
 
 data_filter(undefined) -> undefined;

From 35e347aec82cc9ad2ed15dba6972f27406488ac6 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 8 Aug 2022 17:48:39 +0800
Subject: [PATCH 024/232] feat: refine influxdb bridge conf

Consistent influxdb line protocol config to raw syntax format.
See also
[InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and
[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
---
 .../i18n/emqx_ee_bridge_influxdb.conf         |  56 +++----
 .../src/emqx_ee_bridge_influxdb.erl           |  99 +++++++++--
 .../src/emqx_ee_connector_influxdb.erl        | 157 +++++++++++-------
 3 files changed, 203 insertions(+), 109 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
index 3930825e5..701608721 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -18,45 +18,31 @@ will be forwarded.
                 zh: "本地 Topic"
             }
     }
-    measurement {
+    write_syntax {
         desc {
-            en: """The measurement name to be forwarded to the InfluxDB. Placeholders supported."""
-            zh: """要转发到 InfluxDB 的 Measurement 名称，支持占位符"""
+            en: """
+Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeHolder supported.
+See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and
+[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) </br>
+TLDR:
+```
+<measurement>[,<tag_key>=<tag_value>[,<tag_key>=<tag_value>]] <field_key>=<field_value>[,<field_key>=<field_value>] [<timestamp>]
+```
+"""
+            zh: """
+使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符</br>
+参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及
+[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) </br>
+TLDR:
+```
+<measurement>[,<tag_key>=<tag_value>[,<tag_key>=<tag_value>]] <field_key>=<field_value>[,<field_key>=<field_value>] [<timestamp>]
+```
+"""
             }
         label {
-            en: "Measurement"
-            zh: "Measurement"
-        }
-    }
-    timestamp {
-        desc {
-            en: """The timestamp to be forwarded to the InfluxDB. Placeholders supported. Default is message timestamp"""
-            zh: """要转发到 InfluxDB 的时间戳，支持占位符。默认使用消息的时间戳"""
+            en: "write_syntax"
+            zh: "写语句"
             }
-        label {
-            en: "Timestamp"
-            zh: "Timestamp"
-        }
-    }
-    tags {
-        desc {
-            en: """The tags to be forwarded to the InfluxDB. Placeholders supported."""
-            zh: """要转发到 InfluxDB 的 Tags 数据内容，支持占位符"""
-            }
-        label {
-            en: "Tags"
-            zh: "Tags"
-        }
-    }
-    fields {
-        desc {
-            en: """The fields to be forwarded to the InfluxDB. Placeholders supported."""
-            zh: """要转发到 InfluxDB 的 fields 数据内容，支持占位符"""
-            }
-        label {
-            en: "Fields"
-            zh: "Fields"
-        }
     }
     config_enable {
         desc {
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index a55d9d47a..3a6787af6 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -3,9 +3,10 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_bridge_influxdb).
 
+-include("emqx_ee_bridge.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include("emqx_ee_bridge.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
@@ -55,13 +56,10 @@ values(Protocol, post) ->
         enable => true,
         direction => egress,
         local_topic => <<"local/topic/#">>,
-        measurement => <<"${topic}">>,
-        tags => #{<<"clientid">> => <<"${clientid}">>},
-        fields => #{
-            <<"payload">> => <<"${payload}">>,
-            <<"int_value">> => [int, <<"${payload.int_key}">>],
-            <<"uint_value">> => [uint, <<"${payload.uint_key}">>]
-        }
+        write_syntax =>
+            <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
+                "${clientid}_int_value=${payload.int_key}i,", "uint_value=${payload.uint_key}u,",
+                "bool=${payload.bool}">>
     };
 values(Protocol, put) ->
     values(Protocol, post).
@@ -77,13 +75,7 @@ fields(basic) ->
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
         {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
-        {measurement, mk(binary(), #{desc => ?DESC("measurement"), required => true})},
-        {timestamp,
-            mk(binary(), #{
-                desc => ?DESC("timestamp"), default => <<"${timestamp}">>, required => false
-            })},
-        {tags, mk(map(), #{desc => ?DESC("tags"), required => false})},
-        {fields, mk(map(), #{desc => ?DESC("fields"), required => true})}
+        {write_syntax, fun write_syntax/1}
     ];
 fields("post_udp") ->
     method_fileds(post, influxdb_udp);
@@ -142,3 +134,80 @@ desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for HStream using `", string:to_upper(Method), "` method."];
 desc(_) ->
     undefined.
+
+write_syntax(type) ->
+    list();
+write_syntax(required) ->
+    true;
+write_syntax(validator) ->
+    [?NOT_EMPTY("the value of the field 'write_syntax' cannot be empty")];
+write_syntax(converter) ->
+    fun converter_influx_lines/1;
+write_syntax(desc) ->
+    ?DESC("write_syntax");
+write_syntax(_) ->
+    undefined.
+
+converter_influx_lines(RawLines) ->
+    Lines = string:tokens(str(RawLines), "\n"),
+    lists:reverse(lists:foldl(fun converter_influx_line/2, [], Lines)).
+
+converter_influx_line(Line, AccIn) ->
+    case string:tokens(str(Line), " ") of
+        [MeasurementAndTags, Fields, Timestamp] ->
+            {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
+            [
+                #{
+                    measurement => Measurement,
+                    tags => kv_pairs(Tags),
+                    fields => kv_pairs(string:tokens(Fields, ",")),
+                    timestamp => Timestamp
+                }
+                | AccIn
+            ];
+        [MeasurementAndTags, Fields] ->
+            {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
+            %% TODO: fix here both here and influxdb driver.
+            %% Default value should evaluated by InfluxDB.
+            [
+                #{
+                    measurement => Measurement,
+                    tags => kv_pairs(Tags),
+                    fields => kv_pairs(string:tokens(Fields, ",")),
+                    timestamp => "${timestamp}"
+                }
+                | AccIn
+            ];
+        _ ->
+            throw("Bad InfluxDB Line Protocol schema")
+    end.
+
+split_measurement_and_tags(Subject) ->
+    case string:tokens(Subject, ",") of
+        [] ->
+            throw("Bad Measurement schema");
+        [Measurement] ->
+            {Measurement, []};
+        [Measurement | Tags] ->
+            {Measurement, Tags}
+    end.
+
+kv_pairs(Pairs) ->
+    kv_pairs(Pairs, []).
+kv_pairs([], Acc) ->
+    lists:reverse(Acc);
+kv_pairs([Pair | Rest], Acc) ->
+    case string:tokens(Pair, "=") of
+        [K, V] ->
+            %% Reduplicated keys will be overwritten. Follows InfluxDB Line Protocol.
+            kv_pairs(Rest, [{K, V} | Acc]);
+        _ ->
+            throw(io_lib:format("Bad InfluxDB Line Protocol Key Value pair: ~p", Pair))
+    end.
+
+str(A) when is_atom(A) ->
+    atom_to_list(A);
+str(B) when is_binary(B) ->
+    binary_to_list(B);
+str(S) when is_list(S) ->
+    S.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index f0e5cd333..aca92e791 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -191,10 +191,7 @@ do_start_client(
     ClientConfig,
     Config = #{
         egress := #{
-            measurement := Measurement,
-            timestamp := Timestamp,
-            tags := Tags,
-            fields := Fields
+            write_syntax := Lines
         }
     }
 ) ->
@@ -204,10 +201,7 @@ do_start_client(
                 true ->
                     State = #{
                         client => Client,
-                        measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement),
-                        timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp),
-                        tags => to_tags_config(Tags),
-                        fields => to_fields_config(Fields)
+                        write_syntax => to_config(Lines)
                     },
                     ?SLOG(info, #{
                         msg => "starting influxdb connector success",
@@ -306,46 +300,65 @@ ssl_config(SSL = #{enable := true}) ->
 %% Query
 
 do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client}) ->
-    case data_to_point(Data, State) of
-        {ok, Point} ->
-            case influxdb:write(Client, [Point]) of
-                ok ->
-                    ?SLOG(debug, #{
-                        msg => "influxdb write point success",
-                        connector => InstId,
-                        point => Point
-                    }),
-                    emqx_resource:query_success(AfterQuery);
-                {error, Reason} ->
-                    ?SLOG(error, #{
-                        msg => "influxdb write point failed",
-                        connector => InstId,
-                        reason => Reason
-                    }),
-                    emqx_resource:query_failed(AfterQuery)
-            end;
-        {error, Reason} ->
+    {Points, Errs} = data_to_points(Data, State),
+    lists:foreach(
+        fun({error, Reason}) ->
             ?SLOG(error, #{
                 msg => "influxdb trans point failed",
                 connector => InstId,
                 reason => Reason
+            })
+        end,
+        Errs
+    ),
+    case influxdb:write(Client, Points) of
+        ok ->
+            ?SLOG(debug, #{
+                msg => "influxdb write point success",
+                connector => InstId,
+                points => Points
             }),
-            {error, Reason}
+            emqx_resource:query_success(AfterQuery);
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "influxdb write point failed",
+                connector => InstId,
+                reason => Reason
+            }),
+            emqx_resource:query_failed(AfterQuery)
     end.
 
 %% -------------------------------------------------------------------------------------------------
 %% Tags & Fields Config Trans
 
-to_tags_config(Tags) ->
-    maps:fold(fun to_maps_config/3, #{}, Tags).
+to_config(Lines) ->
+    to_config(Lines, []).
 
-to_fields_config(Fields) ->
-    maps:fold(fun to_maps_config/3, #{}, Fields).
+to_config([], Acc) ->
+    lists:reverse(Acc);
+to_config(
+    [
+        #{
+            measurement := Measurement,
+            timestamp := Timestamp,
+            tags := Tags,
+            fields := Fields
+        }
+        | Rest
+    ],
+    Acc
+) ->
+    Res = #{
+        measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement),
+        timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp),
+        tags => to_kv_config(Tags),
+        fields => to_kv_config(Fields)
+    },
+    to_config(Rest, [Res | Acc]).
+
+to_kv_config(KVfields) ->
+    maps:fold(fun to_maps_config/3, #{}, proplists:to_map(KVfields)).
 
-to_maps_config(K, [IntType, V], Res) when IntType == <<"int">> orelse IntType == <<"uint">> ->
-    NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
-    NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)),
-    Res#{NK => {binary_to_atom(IntType, utf8), NV}};
 to_maps_config(K, V, Res) ->
     NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
     NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)),
@@ -353,14 +366,24 @@ to_maps_config(K, V, Res) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% Tags & Fields Data Trans
-data_to_point(
+data_to_points(Data, #{write_syntax := Lines}) ->
+    lines_to_points(Data, Lines, [], []).
+
+lines_to_points(_, [], Points, Errs) ->
+    {Points, Errs};
+lines_to_points(
     Data,
-    #{
-        measurement := Measurement,
-        timestamp := Timestamp,
-        tags := Tags,
-        fields := Fields
-    }
+    [
+        #{
+            measurement := Measurement,
+            timestamp := Timestamp,
+            tags := Tags,
+            fields := Fields
+        }
+        | Rest
+    ],
+    ResAcc,
+    ErrAcc
 ) ->
     TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
     case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of
@@ -373,24 +396,11 @@ data_to_point(
                 tags => EncodeTags,
                 fields => EncodeFields
             },
-            {ok, Point};
+            lines_to_points(Data, Rest, [Point | ResAcc], ErrAcc);
         BadTimestamp ->
-            {error, {bad_timestamp, BadTimestamp}}
+            lines_to_points(Data, Rest, ResAcc, [{error, {bad_timestamp, BadTimestamp}} | ErrAcc])
     end.
 
-maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse IntType == uint ->
-    KTransOptions = #{return => full_binary},
-    VTransOptions = #{return => rawlist, var_trans => fun data_filter/1},
-    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions),
-    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions),
-    case {NK, NV} of
-        {[undefined], _} ->
-            {Data, Res};
-        {_, [undefined]} ->
-            {Data, Res};
-        {_, [IntV]} when is_integer(IntV) ->
-            {Data, Res#{NK => {IntType, IntV}}}
-    end;
 maps_config_to_data(K, V, {Data, Res}) ->
     KTransOptions = #{return => full_binary},
     VTransOptions = #{return => rawlist, var_trans => fun data_filter/1},
@@ -402,9 +412,38 @@ maps_config_to_data(K, V, {Data, Res}) ->
         {_, [undefined]} ->
             {Data, Res};
         _ ->
-            {Data, Res#{NK => NV}}
+            {Data, Res#{NK => value_type(NV)}}
     end.
 
+value_type([Int, <<"i">>]) when
+    is_integer(Int)
+->
+    {int, Int};
+value_type([UInt, <<"u">>]) ->
+    {uint, UInt};
+value_type([<<"t">>]) ->
+    't';
+value_type([<<"T">>]) ->
+    'T';
+value_type([<<"true">>]) ->
+    'true';
+value_type([<<"TRUE">>]) ->
+    'TRUE';
+value_type([<<"True">>]) ->
+    'True';
+value_type([<<"f">>]) ->
+    'f';
+value_type([<<"F">>]) ->
+    'F';
+value_type([<<"false">>]) ->
+    'false';
+value_type([<<"FALSE">>]) ->
+    'FALSE';
+value_type([<<"False">>]) ->
+    'False';
+value_type(Val) ->
+    Val.
+
 data_filter(undefined) -> undefined;
 data_filter(Int) when is_integer(Int) -> Int;
 data_filter(Number) when is_number(Number) -> Number;

From 7305c3c04c89996abdc74bc96979adaa72dee52f Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 8 Aug 2022 18:22:46 +0800
Subject: [PATCH 025/232] fix(influxdb): parsed boolean type match

---
 lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index aca92e791..4a613e124 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -425,7 +425,7 @@ value_type([<<"t">>]) ->
     't';
 value_type([<<"T">>]) ->
     'T';
-value_type([<<"true">>]) ->
+value_type([true]) ->
     'true';
 value_type([<<"TRUE">>]) ->
     'TRUE';
@@ -435,7 +435,7 @@ value_type([<<"f">>]) ->
     'f';
 value_type([<<"F">>]) ->
     'F';
-value_type([<<"false">>]) ->
+value_type([false]) ->
     'false';
 value_type([<<"FALSE">>]) ->
     'FALSE';

From d51ca2672ac470dbd5fcc53cbf61ab233d505315 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 9 Aug 2022 09:49:14 +0800
Subject: [PATCH 026/232] fix: spellcheck and update checker repo version

---
 .github/workflows/build_slim_packages.yaml      |  2 +-
 ...tream.conf => emqx_ee_bridge_hstreamdb.conf} |  0
 .../i18n/emqx_ee_bridge_influxdb.conf           |  3 ++-
 .../src/emqx_ee_bridge_influxdb.erl             |  8 +++++++-
 ...am.conf => emqx_ee_connector_hstreamdb.conf} | 12 +++++++++++-
 .../i18n/emqx_ee_connector_influxdb.conf        | 17 ++++++++---------
 .../src/emqx_ee_connector_hstreamdb.erl         |  4 ++++
 .../src/emqx_ee_connector_influxdb.erl          |  9 +++++++++
 scripts/spellcheck                              |  2 +-
 9 files changed, 43 insertions(+), 14 deletions(-)
 rename lib-ee/emqx_ee_bridge/i18n/{emqx_ee_bridge_hstream.conf => emqx_ee_bridge_hstreamdb.conf} (100%)
 rename lib-ee/emqx_ee_connector/i18n/{emqx_ee_connector_hstream.conf => emqx_ee_connector_hstreamdb.conf} (87%)

diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 27f43d0ea..488189d81 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -205,7 +205,7 @@ jobs:
         - emqx
         - emqx-enterprise
     runs-on: aws-amd64
-    container: "ghcr.io/emqx/emqx-schema-validate:0.3.3"
+    container: "ghcr.io/emqx/emqx-schema-validate:0.3.5"
     steps:
     - uses: actions/download-artifact@v2
       name: Download schema dump
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf
similarity index 100%
rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf
rename to lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
index 701608721..ffd0b66a0 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -21,7 +21,7 @@ will be forwarded.
     write_syntax {
         desc {
             en: """
-Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeHolder supported.
+Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported.
 See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and
 [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) </br>
 TLDR:
@@ -97,6 +97,7 @@ TLDR:
             zh: "桥接名字"
         }
     }
+
     desc_connector {
         desc {
                 en: """Generic configuration for the connector."""
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 3a6787af6..490abfc09 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -131,7 +131,13 @@ type_name_field(Type) ->
 desc("config") ->
     ?DESC("desc_config");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
-    ["Configuration for HStream using `", string:to_upper(Method), "` method."];
+    ["Configuration for InfluxDB using `", string:to_upper(Method), "` method."];
+desc(influxdb_udp) ->
+    ?DESC(emqx_ee_connector_influxdb, "influxdb_udp");
+desc(influxdb_api_v1) ->
+    ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v1");
+desc(influxdb_api_v2) ->
+    ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v2");
 desc(_) ->
     undefined.
 
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf
similarity index 87%
rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
rename to lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf
index dd9659aba..0826c8f0c 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf
@@ -1,5 +1,15 @@
-
 emqx_ee_connector_hstreamdb {
+    config {
+        desc {
+                en: "HStreamDB connection config"
+                zh: "HStreamDB 连接配置。"
+        }
+        label: {
+                en: "Connection config"
+                zh: "连接配置"
+            }
+    }
+
     type {
         desc {
                 en: "The Connector Type."
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
index a909c9a72..7e223b9b7 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
@@ -1,4 +1,3 @@
-
 emqx_ee_connector_influxdb {
     type {
         desc {
@@ -43,7 +42,7 @@ emqx_ee_connector_influxdb {
     }
     protocol {
         desc {
-                en: """InfluxDB protocol. UDP or HTTP API or HTTP API V2"""
+                en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2"""
                 zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2"""
         }
         label: {
@@ -51,9 +50,9 @@ emqx_ee_connector_influxdb {
                 zh: """协议"""
             }
     }
-    protocol_udp {
+    influxdb_udp {
         desc {
-                en: """InfluxDB protocol."""
+                en: """InfluxDB's UDP protocol."""
                 zh: """InfluxDB UDP 协议"""
         }
         label: {
@@ -61,9 +60,9 @@ emqx_ee_connector_influxdb {
                 zh: """UDP 协议"""
             }
     }
-    protocol_api_v1 {
+    influxdb_api_v1 {
         desc {
-                en: """InfluxDB protocol. Support InfluxDB v1.8 and before."""
+                en: """InfluxDB's protocol. Support InfluxDB v1.8 and before."""
                 zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本"""
         }
         label: {
@@ -71,9 +70,9 @@ emqx_ee_connector_influxdb {
                 zh: """HTTP API 协议"""
             }
     }
-    protocol_api_v2 {
+    influxdb_api_v2 {
         desc {
-                en: """InfluxDB protocol. Support InfluxDB v2.0 and after."""
+                en: """InfluxDB's protocol. Support InfluxDB v2.0 and after."""
                 zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本"""
         }
         label: {
@@ -123,7 +122,7 @@ emqx_ee_connector_influxdb {
     }
     org {
         desc {
-                en: """InfluxDB organization name."""
+                en: """Organization name of InfluxDB."""
                 zh: """InfluxDB 组织名称。"""
         }
         label: {
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
index d0bbed136..8ee37cd8a 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
@@ -27,6 +27,7 @@
     namespace/0,
     roots/0,
     fields/1,
+    desc/1,
     connector_examples/1
 ]).
 
@@ -123,6 +124,9 @@ values(put) ->
 values(_) ->
     #{}.
 
+desc(config) ->
+    ?DESC("config").
+
 %% -------------------------------------------------------------------------------------------------
 %% internal functions
 start_client(InstId, Config) ->
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 4a613e124..9582f1729 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -22,6 +22,7 @@
 -export([
     namespace/0,
     fields/1,
+    desc/1,
     connector_examples/1
 ]).
 
@@ -161,6 +162,14 @@ values(api_v2, put) ->
         token => <<"my_token">>,
         ssl => #{enable => false}
     }.
+
+desc(influxdb_udp) ->
+    ?DESC("influxdb_udp");
+desc(influxdb_api_v1) ->
+    ?DESC("influxdb_api_v1");
+desc(influxdb_api_v2) ->
+    ?DESC("influxdb_api_v2").
+
 %% -------------------------------------------------------------------------------------------------
 %% internal functions
 
diff --git a/scripts/spellcheck b/scripts/spellcheck
index 51d8d2907..f8af8c3f6 100755
--- a/scripts/spellcheck
+++ b/scripts/spellcheck
@@ -7,7 +7,7 @@ else
     SCHEMA="$1"
 fi
 
-docker run -d --name langtool "ghcr.io/emqx/emqx-schema-validate:0.3.3"
+docker run -d --name langtool "ghcr.io/emqx/emqx-schema-validate:0.3.5"
 
 docker exec -i langtool ./emqx_schema_validate - < "${SCHEMA}"
 success="$?"

From 9d17bf31f812495f261b696e1d041533804f1ad7 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 9 Aug 2022 14:28:54 +0800
Subject: [PATCH 027/232] fix: InfluxDB api v1 not support uint type

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 490abfc09..83c5a4127 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -49,6 +49,12 @@ conn_bridge_examples(Method) ->
 values(Protocol, get) ->
     maps:merge(values(Protocol, post), ?METRICS_EXAMPLE);
 values(Protocol, post) ->
+    case Protocol of
+        "influxdb_api_v2" ->
+            SupportUint = <<"uint_value=${payload.uint_key}u">>;
+        _ ->
+            SupportUint = <<>>
+    end,
     #{
         type => list_to_atom(Protocol),
         name => <<"demo">>,
@@ -58,7 +64,7 @@ values(Protocol, post) ->
         local_topic => <<"local/topic/#">>,
         write_syntax =>
             <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
-                "${clientid}_int_value=${payload.int_key}i,", "uint_value=${payload.uint_key}u,",
+                "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary,
                 "bool=${payload.bool}">>
     };
 values(Protocol, put) ->

From 12904d797fe0677fc50b487c3b5387a63aa98fb3 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 25 Jul 2022 23:51:23 +0800
Subject: [PATCH 028/232] feat(resource): first commit for
 batching/async/caching mechanism

---
 apps/emqx_resource/include/emqx_resource.hrl  |   1 +
 .../include/emqx_resource_utils.hrl           |   5 +-
 apps/emqx_resource/src/emqx_resource.erl      |  58 +-
 .../src/emqx_resource_manager.erl             |  30 +-
 apps/emqx_resource/src/emqx_resource_sup.erl  |   4 +-
 .../emqx_resource/src/emqx_resource_utils.erl |  17 +
 .../src/emqx_resource_worker.erl              | 252 ++++++
 apps/emqx_resource/src/replayq.erl            | 779 ++++++++++++++++++
 8 files changed, 1091 insertions(+), 55 deletions(-)
 create mode 100644 apps/emqx_resource/src/emqx_resource_utils.erl
 create mode 100644 apps/emqx_resource/src/emqx_resource_worker.erl
 create mode 100644 apps/emqx_resource/src/replayq.erl

diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index dd384af7c..d6f959510 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -54,3 +54,4 @@
 -type after_query_fun() :: {fun((...) -> ok), Args :: [term()]}.
 
 -define(TEST_ID_PREFIX, "_test_:").
+-define(RES_METRICS, resource_metrics).
diff --git a/apps/emqx_resource/include/emqx_resource_utils.hrl b/apps/emqx_resource/include/emqx_resource_utils.hrl
index 8d94746eb..3df64b1e5 100644
--- a/apps/emqx_resource/include/emqx_resource_utils.hrl
+++ b/apps/emqx_resource/include/emqx_resource_utils.hrl
@@ -15,7 +15,7 @@
 %%--------------------------------------------------------------------
 
 -define(SAFE_CALL(_EXP_),
-    ?SAFE_CALL(_EXP_, ok)
+    ?SAFE_CALL(_EXP_, {error, {_EXCLASS_, _EXCPTION_, _ST_}})
 ).
 
 -define(SAFE_CALL(_EXP_, _EXP_ON_FAIL_),
@@ -24,8 +24,7 @@
             (_EXP_)
         catch
             _EXCLASS_:_EXCPTION_:_ST_ ->
-                _EXP_ON_FAIL_,
-                {error, {_EXCLASS_, _EXCPTION_, _ST_}}
+                _EXP_ON_FAIL_
         end
     end()
 ).
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 33f0d0a3d..793b9f446 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -83,8 +83,7 @@
     stop/1,
     %% query the instance
     query/2,
-    %% query the instance with after_query()
-    query/3
+    query_async/3
 ]).
 
 %% Direct calls to the callback module
@@ -111,6 +110,8 @@
     list_group_instances/1
 ]).
 
+-export([inc_metrics_funcs/1, inc_success/1, inc_failed/1]).
+
 -optional_callbacks([
     on_query/4,
     on_get_status/2
@@ -150,16 +151,16 @@ is_resource_mod(Module) ->
 
 -spec query_success(after_query()) -> ok.
 query_success(undefined) -> ok;
-query_success({OnSucc, _}) -> apply_query_after_calls(OnSucc).
+query_success({OnSucc, _}) -> exec_query_after_calls(OnSucc).
 
 -spec query_failed(after_query()) -> ok.
 query_failed(undefined) -> ok;
-query_failed({_, OnFailed}) -> apply_query_after_calls(OnFailed).
+query_failed({_, OnFailed}) -> exec_query_after_calls(OnFailed).
 
-apply_query_after_calls(Funcs) ->
+exec_query_after_calls(Funcs) ->
     lists:foreach(
-        fun({Fun, Args}) ->
-            safe_apply(Fun, Args)
+        fun({Fun, Arg}) ->
+            emqx_resource_utils:safe_exec(Fun, Arg)
         end,
         Funcs
     ).
@@ -243,29 +244,12 @@ reset_metrics(ResId) ->
 %% =================================================================================
 -spec query(resource_id(), Request :: term()) -> Result :: term().
 query(ResId, Request) ->
-    query(ResId, Request, inc_metrics_funcs(ResId)).
+    emqx_resource_worker:query(ResId, Request).
 
-%% same to above, also defines what to do when the Module:on_query success or failed
-%% it is the duty of the Module to apply the `after_query()` functions.
--spec query(resource_id(), Request :: term(), after_query()) -> Result :: term().
-query(ResId, Request, AfterQuery) ->
-    case emqx_resource_manager:ets_lookup(ResId) of
-        {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
-            %% the resource state is readonly to Module:on_query/4
-            %% and the `after_query()` functions should be thread safe
-            ok = emqx_metrics_worker:inc(resource_metrics, ResId, matched),
-            try
-                Mod:on_query(ResId, Request, AfterQuery, ResourceState)
-            catch
-                Err:Reason:ST ->
-                    emqx_metrics_worker:inc(resource_metrics, ResId, exception),
-                    erlang:raise(Err, Reason, ST)
-            end;
-        {ok, _Group, _Data} ->
-            query_error(not_connected, <<"resource not connected">>);
-        {error, not_found} ->
-            query_error(not_found, <<"resource not found">>)
-    end.
+-spec query_async(resource_id(), Request :: term(), emqx_resource_worker:reply_fun()) ->
+    ok.
+query_async(ResId, Request, ReplyFun) ->
+    emqx_resource_worker:query_async(ResId, Request, ReplyFun).
 
 -spec start(resource_id()) -> ok | {error, Reason :: term()}.
 start(ResId) ->
@@ -429,16 +413,16 @@ check_and_do(ResourceType, RawConfig, Do) when is_function(Do) ->
 
 %% =================================================================================
 
+inc_success(ResId) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ResId, success).
+
+inc_failed(ResId) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ResId, failed).
+
 filter_instances(Filter) ->
     [Id || #{id := Id, mod := Mod} <- list_instances_verbose(), Filter(Id, Mod)].
 
 inc_metrics_funcs(ResId) ->
-    OnFailed = [{fun emqx_metrics_worker:inc/3, [resource_metrics, ResId, failed]}],
-    OnSucc = [{fun emqx_metrics_worker:inc/3, [resource_metrics, ResId, success]}],
+    OnSucc = [{fun ?MODULE:inc_success/1, ResId}],
+    OnFailed = [{fun ?MODULE:inc_failed/1, ResId}],
     {OnSucc, OnFailed}.
-
-safe_apply(Func, Args) ->
-    ?SAFE_CALL(erlang:apply(Func, Args)).
-
-query_error(Reason, Msg) ->
-    {error, {?MODULE, #{reason => Reason, msg => Msg}}}.
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 82be9c58f..f1fa36173 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -109,9 +109,9 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
     % The state machine will make the actual call to the callback/resource module after init
     ok = emqx_resource_manager_sup:ensure_child(MgrId, ResId, Group, ResourceType, Config, Opts),
     ok = emqx_metrics_worker:create_metrics(
-        resource_metrics,
+        ?RES_METRICS,
         ResId,
-        [matched, success, failed, exception],
+        [matched, success, failed, exception, resource_error],
         [matched]
     ),
     case maps:get(start_after_created, Opts, true) of
@@ -207,12 +207,12 @@ ets_lookup(ResId) ->
 
 %% @doc Get the metrics for the specified resource
 get_metrics(ResId) ->
-    emqx_metrics_worker:get_metrics(resource_metrics, ResId).
+    emqx_metrics_worker:get_metrics(?RES_METRICS, ResId).
 
 %% @doc Reset the metrics for the specified resource
 -spec reset_metrics(resource_id()) -> ok.
 reset_metrics(ResId) ->
-    emqx_metrics_worker:reset_metrics(resource_metrics, ResId).
+    emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId).
 
 %% @doc Returns the data for all resources
 -spec list_all() -> [resource_data()] | [].
@@ -298,8 +298,7 @@ handle_event({call, From}, stop, stopped, _Data) ->
     {keep_state_and_data, [{reply, From, ok}]};
 handle_event({call, From}, stop, _State, Data) ->
     Result = stop_resource(Data),
-    UpdatedData = Data#data{status = disconnected},
-    {next_state, stopped, UpdatedData, [{reply, From, Result}]};
+    {next_state, stopped, Data, [{reply, From, Result}]};
 % Called when a resource is to be stopped and removed.
 handle_event({call, From}, {remove, ClearMetrics}, _State, Data) ->
     handle_remove_event(From, ClearMetrics, Data);
@@ -315,9 +314,10 @@ handle_event({call, From}, health_check, _State, Data) ->
     handle_manually_health_check(From, Data);
 % State: CONNECTING
 handle_event(enter, _OldState, connecting, Data) ->
+    UpdatedData = Data#data{status = connected},
     insert_cache(Data#data.id, Data#data.group, Data),
     Actions = [{state_timeout, 0, health_check}],
-    {keep_state_and_data, Actions};
+    {keep_state, UpdatedData, Actions};
 handle_event(internal, start_resource, connecting, Data) ->
     start_resource(Data, undefined);
 handle_event(state_timeout, health_check, connecting, Data) ->
@@ -326,22 +326,24 @@ handle_event(state_timeout, health_check, connecting, Data) ->
 %% The connected state is entered after a successful on_start/2 of the callback mod
 %% and successful health_checks
 handle_event(enter, _OldState, connected, Data) ->
-    insert_cache(Data#data.id, Data#data.group, Data),
+    UpdatedData = Data#data{status = connected},
+    insert_cache(Data#data.id, Data#data.group, UpdatedData),
     _ = emqx_alarm:deactivate(Data#data.id),
     Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}],
-    {next_state, connected, Data, Actions};
+    {next_state, connected, UpdatedData, Actions};
 handle_event(state_timeout, health_check, connected, Data) ->
     handle_connected_health_check(Data);
 %% State: DISCONNECTED
 handle_event(enter, _OldState, disconnected, Data) ->
-    insert_cache(Data#data.id, Data#data.group, Data),
-    handle_disconnected_state_enter(Data);
+    UpdatedData = Data#data{status = disconnected},
+    insert_cache(Data#data.id, Data#data.group, UpdatedData),
+    handle_disconnected_state_enter(UpdatedData);
 handle_event(state_timeout, auto_retry, disconnected, Data) ->
     start_resource(Data, undefined);
 %% State: STOPPED
 %% The stopped state is entered after the resource has been explicitly stopped
 handle_event(enter, _OldState, stopped, Data) ->
-    UpdatedData = Data#data{status = disconnected},
+    UpdatedData = Data#data{status = stopped},
     insert_cache(Data#data.id, Data#data.group, UpdatedData),
     {next_state, stopped, UpdatedData};
 % Ignore all other events
@@ -415,7 +417,7 @@ handle_disconnected_state_enter(Data) ->
 handle_remove_event(From, ClearMetrics, Data) ->
     stop_resource(Data),
     case ClearMetrics of
-        true -> ok = emqx_metrics_worker:clear_metrics(resource_metrics, Data#data.id);
+        true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id);
         false -> ok
     end,
     {stop_and_reply, normal, [{reply, From, ok}]}.
@@ -433,7 +435,7 @@ start_resource(Data, From) ->
             _ = maybe_alarm(disconnected, Data#data.id),
             %% Keep track of the error reason why the connection did not work
             %% so that the Reason can be returned when the verification call is made.
-            UpdatedData = Data#data{status = disconnected, error = Reason},
+            UpdatedData = Data#data{error = Reason},
             Actions = maybe_reply([], From, Err),
             {next_state, disconnected, UpdatedData, Actions}
     end.
diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl
index 1120723c3..84458f0d5 100644
--- a/apps/emqx_resource/src/emqx_resource_sup.erl
+++ b/apps/emqx_resource/src/emqx_resource_sup.erl
@@ -15,6 +15,8 @@
 %%--------------------------------------------------------------------
 -module(emqx_resource_sup).
 
+-include("emqx_resource.hrl").
+
 -behaviour(supervisor).
 
 -export([start_link/0]).
@@ -29,7 +31,7 @@ start_link() ->
 
 init([]) ->
     SupFlags = #{strategy => one_for_one, intensity => 10, period => 10},
-    Metrics = emqx_metrics_worker:child_spec(resource_metrics),
+    Metrics = emqx_metrics_worker:child_spec(?RES_METRICS),
 
     ResourceManager =
         #{
diff --git a/apps/emqx_resource/src/emqx_resource_utils.erl b/apps/emqx_resource/src/emqx_resource_utils.erl
new file mode 100644
index 000000000..715691d2a
--- /dev/null
+++ b/apps/emqx_resource/src/emqx_resource_utils.erl
@@ -0,0 +1,17 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_resource_utils).
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
new file mode 100644
index 000000000..6c3b05830
--- /dev/null
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -0,0 +1,252 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% An FIFO queue using ETS-ReplayQ as backend.
+
+-module(emqx_resource_worker).
+
+-include("emqx_resource.hrl").
+-include("emqx_resource_utils.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+-behaviour(gen_statem).
+
+-export([
+    start_link/2,
+    query/2,
+    query_async/3,
+    query_mfa/3
+]).
+
+-export([
+    callback_mode/0,
+    init/1
+]).
+
+-export([do/3]).
+
+%% count
+-define(DEFAULT_BATCH_SIZE, 100).
+%% milliseconds
+-define(DEFAULT_BATCH_TIME, 10).
+
+-define(QUERY(FROM, REQUEST), {FROM, REQUEST}).
+-define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}).
+-define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]).
+
+-type id() :: binary().
+-type request() :: term().
+-type result() :: term().
+-type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()}.
+-type from() :: pid() | reply_fun().
+
+-callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) ->
+    {{from(), result()}, NewCbState :: term()}.
+
+callback_mode() -> [state_functions].
+
+start_link(Id, Opts) ->
+    gen_statem:start_link({local, name(Id)}, ?MODULE, {Id, Opts}, []).
+
+-spec query(id(), request()) -> ok.
+query(Id, Request) ->
+    gen_statem:call(name(Id), {query, Request}).
+
+-spec query_async(id(), request(), reply_fun()) -> ok.
+query_async(Id, Request, ReplyFun) ->
+    gen_statem:cast(name(Id), {query, Request, ReplyFun}).
+
+-spec name(id()) -> atom().
+name(Id) ->
+    Mod = atom_to_binary(?MODULE, utf8),
+    <<Mod/binary, ":", Id/binary>>.
+
+disk_cache_dir(Id) ->
+    filename:join([emqx:data_dir(), Id, cache]).
+
+init({Id, Opts}) ->
+    BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE),
+    Queue =
+        case maps:get(cache_enabled, Opts, true) of
+            true -> replayq:open(#{dir => disk_cache_dir(Id), seg_bytes => 10000000});
+            false -> undefined
+        end,
+    St = #{
+        id => Id,
+        batch_enabled => maps:get(batch_enabled, Opts, true),
+        batch_size => BatchSize,
+        batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
+        cache_queue => Queue,
+        acc => [],
+        acc_left => BatchSize,
+        tref => undefined
+    },
+    {ok, do, St}.
+
+do(cast, {query, Request, ReplyFun}, #{batch_enabled := true} = State) ->
+    do_acc(ReplyFun, Request, State);
+do(cast, {query, Request, ReplyFun}, #{batch_enabled := false} = State) ->
+    do_query(ReplyFun, Request, State);
+do({call, From}, {query, Request}, #{batch_enabled := true} = State) ->
+    do_acc(From, Request, State);
+do({call, From}, {query, Request}, #{batch_enabled := false} = State) ->
+    do_query(From, Request, State);
+do(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) ->
+    {keep_state, flush(St#{tref := undefined})};
+do(info, {flush, _Ref}, _St) ->
+    keep_state_and_data;
+do(info, Info, _St) ->
+    ?SLOG(error, #{msg => unexpected_msg, info => Info}),
+    keep_state_and_data.
+
+do_acc(From, Request, #{acc := Acc, acc_left := Left} = St0) ->
+    Acc1 = [?QUERY(From, Request) | Acc],
+    St = St0#{acc := Acc1, acc_left := Left - 1},
+    case Left =< 1 of
+        true -> {keep_state, flush(St)};
+        false -> {keep_state, ensure_flush_timer(St)}
+    end.
+
+do_query(From, Request, #{id := Id, cache_queue := Q0} = St0) ->
+    Result = call_query(Id, Request),
+    Q1 = reply_caller(Id, Q0, ?REPLY(From, Request, Result)),
+    {keep_state, St0#{cache_queue := Q1}}.
+
+flush(#{acc := []} = St) ->
+    St;
+flush(
+    #{
+        id := Id,
+        acc := Batch,
+        batch_size := Size,
+        cache_queue := Q0
+    } = St
+) ->
+    BatchResults = call_batch_query(Id, Batch),
+    Q1 = batch_reply_caller(Id, Q0, BatchResults),
+    cancel_flush_timer(
+        St#{
+            acc_left := Size,
+            acc := [],
+            cache_queue := Q1
+        }
+    ).
+
+maybe_append_cache(undefined, _Request) -> undefined;
+maybe_append_cache(Q, Request) -> replayq:append(Q, Request).
+
+batch_reply_caller(Id, Q, BatchResults) ->
+    lists:foldl(
+        fun(Reply, Q1) ->
+            reply_caller(Id, Q1, Reply)
+        end,
+        Q,
+        BatchResults
+    ).
+
+reply_caller(Id, Q, ?REPLY({ReplyFun, Args}, Request, Result)) when is_function(ReplyFun) ->
+    ?SAFE_CALL(ReplyFun(Result, Args)),
+    handle_query_result(Id, Q, Request, Result);
+reply_caller(Id, Q, ?REPLY(From, Request, Result)) ->
+    gen_statem:reply(From, Result),
+    handle_query_result(Id, Q, Request, Result).
+
+handle_query_result(Id, Q, _Request, ok) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, success),
+    Q;
+handle_query_result(Id, Q, _Request, {ok, _}) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, success),
+    Q;
+handle_query_result(Id, Q, _Request, {error, _}) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
+    Q;
+handle_query_result(Id, Q, Request, {error, {resource_error, #{reason := not_connected}}}) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error),
+    maybe_append_cache(Q, Request);
+handle_query_result(Id, Q, _Request, {error, {resource_error, #{}}}) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error),
+    Q;
+handle_query_result(Id, Q, Request, {error, {exception, _}}) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, exception),
+    maybe_append_cache(Q, Request).
+
+call_query(Id, Request) ->
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+    case emqx_resource_manager:ets_lookup(Id) of
+        {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
+            try Mod:on_query(Id, Request, ResourceState) of
+                Result -> Result
+            catch
+                Err:Reason:ST ->
+                    ModB = atom_to_binary(Mod, utf8),
+                    Msg = <<"call failed, func: ", ModB/binary, ":on_query/3">>,
+                    exception_error(Reason, Msg, {Err, Reason, ST})
+            end;
+        {ok, _Group, #{status := stopped}} ->
+            resource_error(stopped, <<"resource stopped or disabled">>);
+        {ok, _Group, _Data} ->
+            resource_error(not_connected, <<"resource not connected">>);
+        {error, not_found} ->
+            resource_error(not_found, <<"resource not found">>)
+    end.
+
+call_batch_query(Id, Batch) ->
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)),
+    case emqx_resource_manager:ets_lookup(Id) of
+        {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
+            try Mod:on_batch_query(Id, Batch, ResourceState) of
+                BatchResults -> BatchResults
+            catch
+                Err:Reason:ST ->
+                    ModB = atom_to_binary(Mod, utf8),
+                    Msg = <<"call failed, func: ", ModB/binary, ":on_batch_query/3">>,
+                    ?EXPAND(exception_error(Reason, Msg, {Err, Reason, ST}), Batch)
+            end;
+        {ok, _Group, _Data} ->
+            ?EXPAND(resource_error(not_connected, <<"resource not connected">>), Batch);
+        {error, not_found} ->
+            ?EXPAND(resource_error(not_found, <<"resource not found">>), Batch)
+    end.
+
+resource_error(Reason, Msg) ->
+    {error, {resource_error, #{reason => Reason, msg => Msg}}}.
+exception_error(Reason, Msg, Details) ->
+    {error, {exception, #{reason => Reason, msg => Msg, details => Details}}}.
+
+%% ==========================================
+ensure_flush_timer(St = #{tref := undefined, batch_time := T}) ->
+    Ref = make_ref(),
+    TRef = erlang:send_after(T, self(), {flush, Ref}),
+    St#{tref => {TRef, Ref}};
+ensure_flush_timer(St) ->
+    St.
+
+cancel_flush_timer(St = #{tref := undefined}) ->
+    St;
+cancel_flush_timer(St = #{tref := {TRef, _Ref}}) ->
+    _ = erlang:cancel_timer(TRef),
+    St#{tref => undefined}.
+
+query_mfa(InsertMode, Request, SyncTimeout) ->
+    {?MODULE, query_fun(InsertMode), query_args(InsertMode, Request, SyncTimeout)}.
+
+query_fun(<<"sync">>) -> query;
+query_fun(<<"async">>) -> query_async.
+
+query_args(<<"sync">>, Request, SyncTimeout) ->
+    [Request, SyncTimeout];
+query_args(<<"async">>, Request, _) ->
+    [Request].
diff --git a/apps/emqx_resource/src/replayq.erl b/apps/emqx_resource/src/replayq.erl
new file mode 100644
index 000000000..437702a5e
--- /dev/null
+++ b/apps/emqx_resource/src/replayq.erl
@@ -0,0 +1,779 @@
+-module(replayq).
+
+-export([open/1, close/1]).
+-export([append/2, pop/2, ack/2, ack_sync/2, peek/1, overflow/1]).
+-export([count/1, bytes/1, is_empty/1, is_mem_only/1]).
+%% exported for troubleshooting
+-export([do_read_items/2]).
+
+%% internal exports for beam reload
+-export([committer_loop/2, default_sizer/1, default_marshaller/1]).
+
+-export_type([config/0, q/0, ack_ref/0, sizer/0, marshaller/0]).
+
+-define(NOTHING_TO_ACK, nothing_to_ack).
+-define(PENDING_ACKS(Ref), {replayq_pending_acks, Ref}).
+
+-type segno() :: pos_integer().
+-type item() :: term().
+-type count() :: non_neg_integer().
+-type id() :: count().
+-type bytes() :: non_neg_integer().
+-type filename() :: file:filename_all().
+-type dir() :: filename().
+-type ack_ref() :: ?NOTHING_TO_ACK | {segno(), ID :: pos_integer()}.
+-type sizer() :: fun((item()) -> bytes()).
+-type marshaller() :: fun((item()) -> binary()).
+
+-type config() :: #{
+    dir => dir(),
+    seg_bytes => bytes(),
+    mem_only => boolean(),
+    max_total_bytes => bytes(),
+    offload => boolean(),
+    sizer => sizer(),
+    marshaller => marshaller()
+}.
+%% writer cursor
+-define(NO_FD, no_fd).
+-type w_cur() :: #{
+    segno := segno(),
+    bytes := bytes(),
+    count := count(),
+    fd := ?NO_FD | file:fd()
+}.
+
+-type stats() :: #{
+    bytes := bytes(),
+    count := count()
+}.
+
+-opaque q() :: #{
+    config := mem_only | config(),
+    stats := stats(),
+    in_mem := queue:queue(in_mem_item()),
+    w_cur => w_cur(),
+    committer => pid(),
+    head_segno => segno(),
+    sizer := sizer(),
+    marshaller => marshaller(),
+    max_total_bytes := bytes()
+}.
+
+-define(LAYOUT_VSN_0, 0).
+-define(LAYOUT_VSN_1, 1).
+-define(MAGIC, 841265288).
+-define(SUFFIX, "replaylog").
+-define(DEFAULT_POP_BYTES_LIMIT, 2000000).
+-define(DEFAULT_POP_COUNT_LIMIT, 1000).
+-define(DEFAULT_REPLAYQ_LIMIT, 2000000000).
+-define(COMMIT(SEGNO, ID, From), {commit, SEGNO, ID, From}).
+-define(NO_COMMIT_HIST, no_commit_hist).
+-define(FIRST_SEGNO, 1).
+-define(NEXT_SEGNO(N), (N + 1)).
+-define(STOP, stop).
+-define(MEM_ONLY_ITEM(Bytes, Item), {Bytes, Item}).
+-define(DISK_CP_ITEM(Id, Bytes, Item), {Id, Bytes, Item}).
+
+-type in_mem_item() ::
+    ?MEM_ONLY_ITEM(bytes(), item())
+    | ?DISK_CP_ITEM(id(), bytes(), item()).
+
+-spec open(config()) -> q().
+open(#{mem_only := true} = C) ->
+    #{
+        stats => #{bytes => 0, count => 0},
+        in_mem => queue:new(),
+        sizer => get_sizer(C),
+        config => mem_only,
+        max_total_bytes => maps:get(max_total_bytes, C, ?DEFAULT_REPLAYQ_LIMIT)
+    };
+open(#{dir := Dir, seg_bytes := _} = Config) ->
+    ok = filelib:ensure_dir(filename:join(Dir, "foo")),
+    Sizer = get_sizer(Config),
+    Marshaller = get_marshaller(Config),
+    IsOffload = is_offload_mode(Config),
+    Q =
+        case delete_consumed_and_list_rest(Dir) of
+            [] ->
+                %% no old segments, start over from zero
+                #{
+                    stats => #{bytes => 0, count => 0},
+                    w_cur => init_writer(Dir, empty, IsOffload),
+                    committer => spawn_committer(?FIRST_SEGNO, Dir),
+                    head_segno => ?FIRST_SEGNO,
+                    in_mem => queue:new()
+                };
+            Segs ->
+                LastSegno = lists:last(Segs),
+                CommitHist = get_commit_hist(Dir),
+                Reader = fun(Seg, Ch) -> read_items(Dir, Seg, Ch, Sizer, Marshaller) end,
+                HeadItems = Reader(hd(Segs), CommitHist),
+                #{
+                    stats => collect_stats(HeadItems, tl(Segs), Reader),
+                    w_cur => init_writer(Dir, LastSegno, IsOffload),
+                    committer => spawn_committer(hd(Segs), Dir),
+                    head_segno => hd(Segs),
+                    in_mem => queue:from_list(HeadItems)
+                }
+        end,
+    Q#{
+        sizer => Sizer,
+        marshaller => Marshaller,
+        config => maps:without([sizer, marshaller], Config),
+        max_total_bytes => maps:get(max_total_bytes, Config, ?DEFAULT_REPLAYQ_LIMIT)
+    }.
+
+-spec close(q() | w_cur()) -> ok | {error, any()}.
+close(#{config := mem_only}) ->
+    ok;
+close(#{w_cur := W_Cur, committer := Pid} = Q) ->
+    MRef = erlang:monitor(process, Pid),
+    Pid ! ?STOP,
+    unlink(Pid),
+    receive
+        {'DOWN', MRef, process, Pid, _Reason} ->
+            ok
+    end,
+    ok = maybe_dump_back_to_disk(Q),
+    do_close(W_Cur).
+
+do_close(#{fd := ?NO_FD}) -> ok;
+do_close(#{fd := Fd}) -> file:close(Fd).
+
+%% In case of offload mode, dump the unacked (and un-popped) on disk
+%% before close. this serves as a best-effort data loss protection
+maybe_dump_back_to_disk(#{config := Config} = Q) ->
+    case is_offload_mode(Config) of
+        true -> dump_back_to_disk(Q);
+        false -> ok
+    end.
+
+dump_back_to_disk(#{
+    config := #{dir := Dir},
+    head_segno := ReaderSegno,
+    in_mem := InMem,
+    marshaller := Marshaller
+}) ->
+    IoData0 = get_unacked(process_info(self(), dictionary), ReaderSegno, Marshaller),
+    Items1 = queue:to_list(InMem),
+    IoData1 = lists:map(fun(?DISK_CP_ITEM(_, _, I)) -> make_iodata(I, Marshaller) end, Items1),
+    %% ensure old segment file is deleted
+    ok = ensure_deleted(filename(Dir, ReaderSegno)),
+    %% rewrite the segment with what's currently in memory
+    IoData = [IoData0, IoData1],
+    case iolist_size(IoData) > 0 of
+        true ->
+            #{fd := Fd} = open_segment(Dir, ReaderSegno),
+            ok = file:write(Fd, [IoData0, IoData1]),
+            ok = file:close(Fd);
+        false ->
+            %% nothing to write
+            ok
+    end.
+
+get_unacked({dictionary, Dict}, ReaderSegno, Marshaller) ->
+    F = fun
+        ({?PENDING_ACKS(AckRef), Items}) ->
+            erase(?PENDING_ACKS(AckRef)),
+            {Segno, Id} = AckRef,
+            Segno =:= ReaderSegno andalso
+                {true, {Id, Items}};
+        (_) ->
+            false
+    end,
+    Pendings0 = lists:filtermap(F, Dict),
+    Pendings = lists:keysort(1, Pendings0),
+    do_get_unacked(Pendings, Marshaller).
+
+do_get_unacked([], _Marshaller) ->
+    [];
+do_get_unacked([{_, Items} | Rest], Marshaller) ->
+    [
+        [make_iodata(I, Marshaller) || I <- Items]
+        | do_get_unacked(Rest, Marshaller)
+    ].
+
+-spec append(q(), [item()]) -> q().
+append(Q, []) ->
+    Q;
+append(
+    #{
+        config := mem_only,
+        in_mem := InMem,
+        stats := #{bytes := Bytes0, count := Count0},
+        sizer := Sizer
+    } = Q,
+    Items0
+) ->
+    {CountDiff, BytesDiff, Items} = transform(false, Items0, Sizer),
+
+    Stats = #{count => Count0 + CountDiff, bytes => Bytes0 + BytesDiff},
+    Q#{
+        stats := Stats,
+        in_mem := append_in_mem(Items, InMem)
+    };
+append(
+    #{
+        config := #{seg_bytes := BytesLimit, dir := Dir} = Config,
+        stats := #{bytes := Bytes0, count := Count0},
+        w_cur := #{count := CountInSeg, segno := WriterSegno} = W_Cur0,
+        head_segno := ReaderSegno,
+        sizer := Sizer,
+        marshaller := Marshaller,
+        in_mem := HeadItems0
+    } = Q,
+    Items0
+) ->
+    IoData = lists:map(fun(I) -> make_iodata(I, Marshaller) end, Items0),
+    {CountDiff, BytesDiff, Items} = transform(CountInSeg + 1, Items0, Sizer),
+    TotalBytes = Bytes0 + BytesDiff,
+    Stats = #{count => Count0 + CountDiff, bytes => TotalBytes},
+    IsOffload = is_offload_mode(Config),
+    W_Cur1 = do_append(W_Cur0, CountDiff, BytesDiff, IoData),
+    W_Cur =
+        case is_segment_full(W_Cur1, TotalBytes, BytesLimit, ReaderSegno, IsOffload) of
+            true ->
+                ok = do_close(W_Cur1),
+                %% get ready for the next append
+                open_segment(Dir, ?NEXT_SEGNO(WriterSegno));
+            false ->
+                W_Cur1
+        end,
+    HeadItems =
+        case ReaderSegno =:= WriterSegno of
+            true -> append_in_mem(Items, HeadItems0);
+            false -> HeadItems0
+        end,
+    Q#{
+        stats := Stats,
+        w_cur := W_Cur,
+        in_mem := HeadItems
+    }.
+
+%% @doc pop out at least one item from the queue.
+%% volume limited by `bytes_limit' and `count_limit'.
+-spec pop(q(), #{bytes_limit => bytes(), count_limit => count()}) ->
+    {q(), ack_ref(), [item()]}.
+pop(Q, Opts) ->
+    Bytes = maps:get(bytes_limit, Opts, ?DEFAULT_POP_BYTES_LIMIT),
+    Count = maps:get(count_limit, Opts, ?DEFAULT_POP_COUNT_LIMIT),
+    true = (Count > 0),
+    pop(Q, Bytes, Count, ?NOTHING_TO_ACK, []).
+
+%% @doc peek the queue front item.
+-spec peek(q()) -> empty | item().
+peek(#{in_mem := HeadItems}) ->
+    case queue:peek(HeadItems) of
+        empty -> empty;
+        {value, ?MEM_ONLY_ITEM(_, Item)} -> Item;
+        {value, ?DISK_CP_ITEM(_, _, Item)} -> Item
+    end.
+
+%% @doc Asynch-ly write the consumed item Segment number + ID to a file.
+-spec ack(q(), ack_ref()) -> ok.
+ack(_, ?NOTHING_TO_ACK) ->
+    ok;
+ack(#{committer := Pid}, {Segno, Id} = AckRef) ->
+    _ = erlang:erase(?PENDING_ACKS(AckRef)),
+    Pid ! ?COMMIT(Segno, Id, false),
+    ok.
+
+%% @hidden Synced ack, for deterministic tests only
+-spec ack_sync(q(), ack_ref()) -> ok.
+ack_sync(_, ?NOTHING_TO_ACK) ->
+    ok;
+ack_sync(#{committer := Pid}, {Segno, Id} = AckRef) ->
+    _ = erlang:erase(?PENDING_ACKS(AckRef)),
+    Ref = make_ref(),
+    Pid ! ?COMMIT(Segno, Id, {self(), Ref}),
+    receive
+        {Ref, ok} -> ok
+    end.
+
+-spec count(q()) -> count().
+count(#{stats := #{count := Count}}) -> Count.
+
+-spec bytes(q()) -> bytes().
+bytes(#{stats := #{bytes := Bytes}}) -> Bytes.
+
+is_empty(#{config := mem_only, in_mem := All}) ->
+    queue:is_empty(All);
+is_empty(
+    #{
+        w_cur := #{segno := WriterSegno},
+        head_segno := ReaderSegno,
+        in_mem := HeadItems
+    } = Q
+) ->
+    Result = ((WriterSegno =:= ReaderSegno) andalso queue:is_empty(HeadItems)),
+    %% assert
+    Result = (count(Q) =:= 0).
+
+%% @doc Returns number of bytes the size of the queue has exceeded
+%% total bytes limit. Result is negative when it is not overflow.
+-spec overflow(q()) -> integer().
+overflow(#{
+    max_total_bytes := MaxTotalBytes,
+    stats := #{bytes := Bytes}
+}) ->
+    Bytes - MaxTotalBytes.
+
+-spec is_mem_only(q()) -> boolean().
+is_mem_only(#{config := mem_only}) ->
+    true;
+is_mem_only(_) ->
+    false.
+
+%% internals =========================================================
+
+transform(Id, Items, Sizer) ->
+    transform(Id, Items, Sizer, 0, 0, []).
+
+transform(_Id, [], _Sizer, Count, Bytes, Acc) ->
+    {Count, Bytes, lists:reverse(Acc)};
+transform(Id, [Item0 | Rest], Sizer, Count, Bytes, Acc) ->
+    Size = Sizer(Item0),
+    {NextId, Item} =
+        case Id of
+            false -> {false, ?MEM_ONLY_ITEM(Size, Item0)};
+            N -> {N + 1, ?DISK_CP_ITEM(Id, Size, Item0)}
+        end,
+    transform(NextId, Rest, Sizer, Count + 1, Bytes + Size, [Item | Acc]).
+
+append_in_mem([], Q) -> Q;
+append_in_mem([Item | Rest], Q) -> append_in_mem(Rest, queue:in(Item, Q)).
+
+pop(Q, _Bytes, 0, AckRef, Acc) ->
+    Result = lists:reverse(Acc),
+    ok = maybe_save_pending_acks(AckRef, Q, Result),
+    {Q, AckRef, Result};
+pop(#{config := Cfg} = Q, Bytes, Count, AckRef, Acc) ->
+    case is_empty(Q) of
+        true ->
+            {Q, AckRef, lists:reverse(Acc)};
+        false when Cfg =:= mem_only ->
+            pop_mem(Q, Bytes, Count, Acc);
+        false ->
+            pop2(Q, Bytes, Count, AckRef, Acc)
+    end.
+
+pop_mem(
+    #{
+        in_mem := InMem,
+        stats := #{count := TotalCount, bytes := TotalBytes} = Stats
+    } = Q,
+    Bytes,
+    Count,
+    Acc
+) ->
+    case queue:out(InMem) of
+        {{value, ?MEM_ONLY_ITEM(Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] ->
+            {Q, ?NOTHING_TO_ACK, lists:reverse(Acc)};
+        {{value, ?MEM_ONLY_ITEM(Sz, Item)}, Rest} ->
+            NewQ = Q#{
+                in_mem := Rest,
+                stats := Stats#{
+                    count := TotalCount - 1,
+                    bytes := TotalBytes - Sz
+                }
+            },
+            pop(NewQ, Bytes - Sz, Count - 1, ?NOTHING_TO_ACK, [Item | Acc])
+    end.
+
+pop2(
+    #{
+        head_segno := ReaderSegno,
+        in_mem := HeadItems,
+        stats := #{count := TotalCount, bytes := TotalBytes} = Stats,
+        w_cur := #{segno := WriterSegno}
+    } = Q,
+    Bytes,
+    Count,
+    AckRef,
+    Acc
+) ->
+    case queue:out(HeadItems) of
+        {{value, ?DISK_CP_ITEM(_, Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] ->
+            %% taking the head item would cause exceeding size limit
+            {Q, AckRef, lists:reverse(Acc)};
+        {{value, ?DISK_CP_ITEM(Id, Sz, Item)}, Rest} ->
+            Q1 = Q#{
+                in_mem := Rest,
+                stats := Stats#{
+                    count := TotalCount - 1,
+                    bytes := TotalBytes - Sz
+                }
+            },
+            %% read the next segment in case current is drained
+            NewQ =
+                case queue:is_empty(Rest) andalso ReaderSegno < WriterSegno of
+                    true -> read_next_seg(Q1);
+                    false -> Q1
+                end,
+            NewAckRef = {ReaderSegno, Id},
+            pop(NewQ, Bytes - Sz, Count - 1, NewAckRef, [Item | Acc])
+    end.
+
+%% due to backward compatibility reasons for the ack api
+%% we ca nnot track pending acks in q() -- reason to use process dictionary
+maybe_save_pending_acks(?NOTHING_TO_ACK, _, _) ->
+    ok;
+maybe_save_pending_acks(AckRef, #{config := Config}, Items) ->
+    case is_offload_mode(Config) of
+        true ->
+            _ = erlang:put(?PENDING_ACKS(AckRef), Items),
+            ok;
+        false ->
+            ok
+    end.
+
+read_next_seg(
+    #{
+        config := #{dir := Dir} = Config,
+        head_segno := ReaderSegno,
+        w_cur := #{segno := WriterSegno, fd := Fd} = WCur0,
+        sizer := Sizer,
+        marshaller := Marshaller
+    } = Q
+) ->
+    NextSegno = ReaderSegno + 1,
+    %% reader has caught up to latest segment
+    case NextSegno =:= WriterSegno of
+        true ->
+            %% force flush to disk so the next read can get all bytes
+            ok = file:sync(Fd);
+        false ->
+            ok
+    end,
+    IsOffload = is_offload_mode(Config),
+    WCur =
+        case IsOffload andalso NextSegno =:= WriterSegno of
+            true ->
+                %% reader has caught up to latest segment in offload mode,
+                %% close the writer's fd. Continue in mem-only mode for the head segment
+                ok = do_close(WCur0),
+                WCur0#{fd := ?NO_FD};
+            false ->
+                WCur0
+        end,
+    NextSegItems = read_items(Dir, NextSegno, ?NO_COMMIT_HIST, Sizer, Marshaller),
+    Q#{
+        head_segno := NextSegno,
+        in_mem := queue:from_list(NextSegItems),
+        w_cur := WCur
+    }.
+
+delete_consumed_and_list_rest(Dir0) ->
+    Dir = unicode:characters_to_list(Dir0),
+    Segnos0 = lists:sort([parse_segno(N) || N <- filelib:wildcard("*." ?SUFFIX, Dir)]),
+    {SegnosToDelete, Segnos} = find_segnos_to_delete(Dir, Segnos0),
+    ok = lists:foreach(fun(Segno) -> ensure_deleted(filename(Dir, Segno)) end, SegnosToDelete),
+    case Segnos of
+        [] ->
+            %% delete commit file in case there is no segments left
+            %% segment number will start from 0 again.
+            ensure_deleted(commit_filename(Dir)),
+            [];
+        X ->
+            X
+    end.
+
+find_segnos_to_delete(Dir, Segnos) ->
+    CommitHist = get_commit_hist(Dir),
+    do_find_segnos_to_delete(Dir, Segnos, CommitHist).
+
+do_find_segnos_to_delete(_Dir, Segnos, ?NO_COMMIT_HIST) ->
+    {[], Segnos};
+do_find_segnos_to_delete(Dir, Segnos0, {CommittedSegno, CommittedId}) ->
+    {SegnosToDelete, Segnos} = lists:partition(fun(N) -> N < CommittedSegno end, Segnos0),
+    case
+        Segnos =/= [] andalso
+            hd(Segnos) =:= CommittedSegno andalso
+            is_all_consumed(Dir, CommittedSegno, CommittedId)
+    of
+        true ->
+            %% assert
+            CommittedSegno = hd(Segnos),
+            %% all items in the oldest segment have been consumed,
+            %% no need to keep this segment
+            {[CommittedSegno | SegnosToDelete], tl(Segnos)};
+        _ ->
+            {SegnosToDelete, Segnos}
+    end.
+
+%% ALL items are consumed if the committed item ID is no-less than the number
+%% of items in this segment
+is_all_consumed(Dir, CommittedSegno, CommittedId) ->
+    CommittedId >= erlang:length(do_read_items(Dir, CommittedSegno)).
+
+ensure_deleted(Filename) ->
+    case file:delete(Filename) of
+        ok -> ok;
+        {error, enoent} -> ok
+    end.
+
+%% The committer writes consumer's acked segmeng number + item ID
+%% to a file. The file is only read at start/restart.
+spawn_committer(ReaderSegno, Dir) ->
+    Name = iolist_to_binary(filename:join([Dir, committer])),
+    %% register a name to avoid having two committers spawned for the same dir
+    RegName = binary_to_atom(Name, utf8),
+    Pid = erlang:spawn_link(fun() -> committer_loop(ReaderSegno, Dir) end),
+    true = erlang:register(RegName, Pid),
+    Pid.
+
+committer_loop(ReaderSegno, Dir) ->
+    receive
+        ?COMMIT(Segno0, Id0, false) ->
+            {Segno, Id} = collect_async_commits(Segno0, Id0),
+            ok = handle_commit(ReaderSegno, Dir, Segno, Id, false),
+            ?MODULE:committer_loop(Segno, Dir);
+        ?COMMIT(Segno, Id, From) ->
+            ok = handle_commit(ReaderSegno, Dir, Segno, Id, From),
+            ?MODULE:committer_loop(Segno, Dir);
+        ?STOP ->
+            ok;
+        Msg ->
+            exit({replayq_committer_unkown_msg, Msg})
+    after 200 ->
+        ?MODULE:committer_loop(ReaderSegno, Dir)
+    end.
+
+handle_commit(ReaderSegno, Dir, Segno, Id, From) ->
+    IoData = io_lib:format("~p.\n", [#{segno => Segno, id => Id}]),
+    ok = do_commit(Dir, IoData),
+    case Segno > ReaderSegno of
+        true ->
+            SegnosToDelete = lists:seq(ReaderSegno, Segno - 1),
+            lists:foreach(fun(N) -> ok = ensure_deleted(filename(Dir, N)) end, SegnosToDelete);
+        false ->
+            ok
+    end,
+    ok = reply_ack_ok(From).
+
+%% Collect async acks which are already sent in the mailbox,
+%% and keep only the last one for the current segment.
+collect_async_commits(Segno, Id) ->
+    receive
+        ?COMMIT(Segno, AnotherId, false) ->
+            collect_async_commits(Segno, AnotherId)
+    after 0 ->
+        {Segno, Id}
+    end.
+
+reply_ack_ok({Pid, Ref}) ->
+    Pid ! {Ref, ok},
+    ok;
+reply_ack_ok(_) ->
+    ok.
+
+get_commit_hist(Dir) ->
+    CommitFile = commit_filename(Dir),
+    case filelib:is_regular(CommitFile) of
+        true ->
+            {ok, [#{segno := Segno, id := Id}]} = file:consult(CommitFile),
+            {Segno, Id};
+        false ->
+            ?NO_COMMIT_HIST
+    end.
+
+do_commit(Dir, IoData) ->
+    TmpName = commit_filename(Dir, "COMMIT.tmp"),
+    Name = commit_filename(Dir),
+    ok = file:write_file(TmpName, IoData),
+    ok = file:rename(TmpName, Name).
+
+commit_filename(Dir) ->
+    commit_filename(Dir, "COMMIT").
+
+commit_filename(Dir, Name) ->
+    filename:join([Dir, Name]).
+
+do_append(
+    #{fd := ?NO_FD, bytes := Bytes0, count := Count0} = Cur,
+    Count,
+    Bytes,
+    _IoData
+) ->
+    %% offload mode, fd is not initialized yet
+    Cur#{
+        bytes => Bytes0 + Bytes,
+        count => Count0 + Count
+    };
+do_append(
+    #{fd := Fd, bytes := Bytes0, count := Count0} = Cur,
+    Count,
+    Bytes,
+    IoData
+) ->
+    ok = file:write(Fd, IoData),
+    Cur#{
+        bytes => Bytes0 + Bytes,
+        count => Count0 + Count
+    }.
+
+read_items(Dir, Segno, CommitHist, Sizer, Marshaller) ->
+    Items0 = do_read_items(Dir, Segno),
+    Items =
+        case CommitHist of
+            ?NO_COMMIT_HIST ->
+                %% no commit hist, return all
+                Items0;
+            {CommitedSegno, _} when CommitedSegno < Segno ->
+                %% committed at an older segment
+                Items0;
+            {Segno, CommittedId} ->
+                %% committed at current segment keep only the tail
+                {_, R} = lists:splitwith(fun({I, _}) -> I =< CommittedId end, Items0),
+                R
+        end,
+    lists:map(
+        fun({Id, Bin}) ->
+            Item = Marshaller(Bin),
+            Size = Sizer(Item),
+            ?DISK_CP_ITEM(Id, Size, Item)
+        end,
+        Items
+    ).
+
+do_read_items(Dir, Segno) ->
+    Filename = filename(Dir, Segno),
+    {ok, Bin} = file:read_file(Filename),
+    case parse_items(Bin, 1, []) of
+        {Items, <<>>} ->
+            Items;
+        {Items, Corrupted} ->
+            error_logger:error_msg(
+                "corrupted replayq log: ~s, skipped ~p bytes",
+                [Filename, size(Corrupted)]
+            ),
+            Items
+    end.
+
+parse_items(<<>>, _Id, Acc) ->
+    {lists:reverse(Acc), <<>>};
+parse_items(
+    <<?LAYOUT_VSN_1:8, ?MAGIC:32/unsigned-integer, CRC:32/unsigned-integer,
+        Size:32/unsigned-integer, Item:Size/binary, Rest/binary>> = All,
+    Id,
+    Acc
+) ->
+    case CRC =:= erlang:crc32(Item) of
+        true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]);
+        false -> {lists:reverse(Acc), All}
+    end;
+parse_items(
+    <<?LAYOUT_VSN_0:8, CRC:32/unsigned-integer, Size:32/unsigned-integer, Item:Size/binary,
+        Rest/binary>> = All,
+    Id,
+    Acc
+) ->
+    case CRC =:= erlang:crc32(Item) andalso Item =/= <<>> of
+        true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]);
+        false -> {lists:reverse(Acc), All}
+    end;
+parse_items(Corrupted, _Id, Acc) ->
+    {lists:reverse(Acc), Corrupted}.
+
+make_iodata(Item0, Marshaller) ->
+    Item = Marshaller(Item0),
+    Size = size(Item),
+    CRC = erlang:crc32(Item),
+    [
+        <<?LAYOUT_VSN_1:8, ?MAGIC:32/unsigned-integer, CRC:32/unsigned-integer,
+            Size:32/unsigned-integer>>,
+        Item
+    ].
+
+collect_stats(HeadItems, SegsOnDisk, Reader) ->
+    ItemF = fun(?DISK_CP_ITEM(_Id, Sz, _Item), {B, C}) ->
+        {B + Sz, C + 1}
+    end,
+    Acc0 = lists:foldl(ItemF, {0, 0}, HeadItems),
+    {Bytes, Count} =
+        lists:foldl(
+            fun(Segno, Acc) ->
+                Items = Reader(Segno, ?NO_COMMIT_HIST),
+                lists:foldl(ItemF, Acc, Items)
+            end,
+            Acc0,
+            SegsOnDisk
+        ),
+    #{bytes => Bytes, count => Count}.
+
+parse_segno(Filename) ->
+    [Segno, ?SUFFIX] = string:tokens(Filename, "."),
+    list_to_integer(Segno).
+
+filename(Dir, Segno) ->
+    Name = lists:flatten(io_lib:format("~10.10.0w." ?SUFFIX, [Segno])),
+    filename:join(Dir, Name).
+
+%% open the current segment for write if it is empty
+%% otherwise rollout to the next segment
+-spec init_writer(dir(), empty | segno(), boolean()) -> w_cur().
+init_writer(_Dir, empty, true) ->
+    %% clean start for offload mode
+    #{fd => ?NO_FD, segno => ?FIRST_SEGNO, bytes => 0, count => 0};
+init_writer(Dir, empty, false) ->
+    open_segment(Dir, ?FIRST_SEGNO);
+init_writer(Dir, Segno, _IsOffload) when is_number(Segno) ->
+    Filename = filename(Dir, Segno),
+    case filelib:file_size(Filename) of
+        0 -> open_segment(Dir, Segno);
+        _ -> open_segment(Dir, ?NEXT_SEGNO(Segno))
+    end.
+
+-spec open_segment(dir(), segno()) -> w_cur().
+open_segment(Dir, Segno) ->
+    Filename = filename(Dir, Segno),
+    %% raw so there is no need to go through the single gen_server file_server
+    {ok, Fd} = file:open(Filename, [raw, read, write, binary, delayed_write]),
+    #{fd => Fd, segno => Segno, bytes => 0, count => 0}.
+
+get_sizer(C) ->
+    maps:get(sizer, C, fun ?MODULE:default_sizer/1).
+
+get_marshaller(C) ->
+    maps:get(marshaller, C, fun ?MODULE:default_marshaller/1).
+
+is_offload_mode(Config) when is_map(Config) ->
+    maps:get(offload, Config, false).
+
+default_sizer(I) when is_binary(I) -> erlang:size(I).
+
+default_marshaller(I) when is_binary(I) -> I.
+
+is_segment_full(
+    #{segno := WriterSegno, bytes := SegmentBytes},
+    TotalBytes,
+    SegmentBytesLimit,
+    ReaderSegno,
+    true
+) ->
+    %% in offload mode, when reader is lagging behind, we try
+    %% writer rolls to a new segment when file size limit is reached
+    %% when reader is reading off from the same segment as writer
+    %% i.e. the in memory queue, only start writing to segment file
+    %% when total bytes (in memory) is reached segment limit
+    %%
+    %% NOTE: we never shrink segment bytes, even when popping out
+    %% from the in-memory queue.
+    case ReaderSegno < WriterSegno of
+        true -> SegmentBytes >= SegmentBytesLimit;
+        false -> TotalBytes >= SegmentBytesLimit
+    end;
+is_segment_full(
+    #{bytes := SegmentBytes},
+    _TotalBytes,
+    SegmentBytesLimit,
+    _ReaderSegno,
+    false
+) ->
+    %% here we check if segment size is greater than segment size limit
+    %% after append based on the assumption that the items are usually
+    %% very small in size comparing to segment size.
+    %% We can change implementation to split items list to avoid
+    %% segment overflow if really necessary
+    SegmentBytes >= SegmentBytesLimit.

From d8d8d674e4ad7314e87926a9a488b5db82008583 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 27 Jul 2022 00:37:17 +0800
Subject: [PATCH 029/232] feat(resource): start emqx_resource_worker in pools

---
 .../src/emqx_resource_manager.erl             |   9 +-
 apps/emqx_resource/src/emqx_resource_sup.erl  |  80 ++++-
 .../src/emqx_resource_worker.erl              | 316 +++++++++++-------
 3 files changed, 278 insertions(+), 127 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index f1fa36173..ee18a6836 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -111,12 +111,15 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
     ok = emqx_metrics_worker:create_metrics(
         ?RES_METRICS,
         ResId,
-        [matched, success, failed, exception, resource_error],
+        [matched, success, failed, exception, resource_down],
         [matched]
     ),
     case maps:get(start_after_created, Opts, true) of
-        true -> wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000));
-        false -> ok
+        true ->
+            ok = emqx_resource_sup:start_workers(ResId, Opts),
+            wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000));
+        false ->
+            ok
     end,
     ok.
 
diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl
index 84458f0d5..14db50d01 100644
--- a/apps/emqx_resource/src/emqx_resource_sup.erl
+++ b/apps/emqx_resource/src/emqx_resource_sup.erl
@@ -19,13 +19,10 @@
 
 -behaviour(supervisor).
 
--export([start_link/0]).
+-export([start_link/0, start_workers/2, stop_workers/2]).
 
 -export([init/1]).
 
-%% set a very large pool size in case all the workers busy
--define(POOL_SIZE, 64).
-
 start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
@@ -43,3 +40,78 @@ init([]) ->
             modules => [emqx_resource_manager_sup]
         },
     {ok, {SupFlags, [Metrics, ResourceManager]}}.
+
+start_workers(ResId, Opts) ->
+    PoolSize = pool_size(Opts),
+    _ = ensure_worker_pool(ResId, hash, [{size, PoolSize}]),
+    lists:foreach(
+        fun(Idx) ->
+            _ = ensure_worker_added(ResId, {ResId, Idx}, Idx),
+            ok = ensure_worker_started(ResId, Idx, Opts)
+        end,
+        lists:seq(1, PoolSize)
+    ).
+
+stop_workers(ResId, Opts) ->
+    PoolSize = pool_size(Opts),
+    lists:foreach(
+        fun(Idx) ->
+            ok = ensure_worker_stopped(ResId, Idx),
+            ok = ensure_worker_removed(ResId, {ResId, Idx})
+        end,
+        lists:seq(1, PoolSize)
+    ),
+    _ = gproc_pool:delete(ResId),
+    ok.
+
+pool_size(Opts) ->
+    maps:get(worker_pool_size, Opts, erlang:system_info(schedulers_online)).
+
+ensure_worker_pool(Pool, Type, Opts) ->
+    try
+        gproc_pool:new(Pool, Type, Opts)
+    catch
+        error:exists -> ok
+    end,
+    ok.
+
+ensure_worker_added(Pool, Name, Slot) ->
+    try
+        gproc_pool:add_worker(Pool, Name, Slot)
+    catch
+        error:exists -> ok
+    end,
+    ok.
+
+ensure_worker_removed(Pool, Name) ->
+    _ = gproc_pool:remove_worker(Pool, Name),
+    ok.
+
+-define(CHILD_ID(MOD, RESID, INDEX), {MOD, RESID, INDEX}).
+ensure_worker_started(ResId, Idx, Opts) ->
+    Mod = emqx_resource_worker,
+    Spec = #{
+        id => ?CHILD_ID(Mod, ResId, Idx),
+        start => {Mod, start_link, [ResId, Idx, Opts]},
+        restart => transient,
+        shutdown => 5000,
+        type => worker,
+        modules => [Mod]
+    },
+    case supervisor:start_child(emqx_resource_sup, Spec) of
+        {ok, _Pid} -> ok;
+        {error, {already_started, _}} -> ok;
+        {error, already_present} -> ok;
+        {error, _} = Err -> Err
+    end.
+
+ensure_worker_stopped(ResId, Idx) ->
+    ChildId = ?CHILD_ID(emqx_resource_worker, ResId, Idx),
+    case supervisor:terminate_child(emqx_resource_sup, ChildId) of
+        ok ->
+            supervisor:delete_child(emqx_resource_sup, ChildId);
+        {error, not_found} ->
+            ok;
+        {error, Reason} ->
+            {error, Reason}
+    end.
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 6c3b05830..3c5c7eefe 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -14,7 +14,8 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% An FIFO queue using ETS-ReplayQ as backend.
+%% This module implements async message sending, disk message queuing,
+%%  and message batching using ReplayQ.
 
 -module(emqx_resource_worker).
 
@@ -25,18 +26,23 @@
 -behaviour(gen_statem).
 
 -export([
-    start_link/2,
-    query/2,
-    query_async/3,
-    query_mfa/3
+    start_link/3,
+    query/3,
+    query_async/4,
+    block/1,
+    resume/1
 ]).
 
 -export([
     callback_mode/0,
-    init/1
+    init/1,
+    terminate/2,
+    code_change/3
 ]).
 
--export([do/3]).
+-export([running/3, blocked/3]).
+
+-define(RESUME_INTERVAL, 15000).
 
 %% count
 -define(DEFAULT_BATCH_SIZE, 100).
@@ -47,72 +53,136 @@
 -define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}).
 -define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]).
 
+-define(RESOURCE_ERROR(Reason, Msg), {error, {resource_error, #{reason => Reason, msg => Msg}}}).
+-define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
+
 -type id() :: binary().
 -type request() :: term().
 -type result() :: term().
--type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()}.
+-type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined.
 -type from() :: pid() | reply_fun().
 
 -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) ->
     {{from(), result()}, NewCbState :: term()}.
 
-callback_mode() -> [state_functions].
+callback_mode() -> [state_functions, state_enter].
 
-start_link(Id, Opts) ->
-    gen_statem:start_link({local, name(Id)}, ?MODULE, {Id, Opts}, []).
+start_link(Id, Index, Opts) ->
+    gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []).
 
--spec query(id(), request()) -> ok.
-query(Id, Request) ->
-    gen_statem:call(name(Id), {query, Request}).
+-spec query(id(), term(), request()) -> ok.
+query(Id, Key, Request) ->
+    gen_statem:call(pick(Id, Key), {query, Request}).
 
--spec query_async(id(), request(), reply_fun()) -> ok.
-query_async(Id, Request, ReplyFun) ->
-    gen_statem:cast(name(Id), {query, Request, ReplyFun}).
+-spec query_async(id(), term(), request(), reply_fun()) -> ok.
+query_async(Id, Key, Request, ReplyFun) ->
+    gen_statem:cast(pick(Id, Key), {query, Request, ReplyFun}).
 
--spec name(id()) -> atom().
-name(Id) ->
-    Mod = atom_to_binary(?MODULE, utf8),
-    <<Mod/binary, ":", Id/binary>>.
+-spec block(pid() | atom()) -> ok.
+block(ServerRef) ->
+    gen_statem:cast(ServerRef, block).
 
-disk_cache_dir(Id) ->
-    filename:join([emqx:data_dir(), Id, cache]).
+-spec resume(pid() | atom()) -> ok.
+resume(ServerRef) ->
+    gen_statem:cast(ServerRef, resume).
 
-init({Id, Opts}) ->
+init({Id, Index, Opts}) ->
+    true = gproc_pool:connect_worker(Id, {Id, Index}),
     BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE),
     Queue =
-        case maps:get(cache_enabled, Opts, true) of
-            true -> replayq:open(#{dir => disk_cache_dir(Id), seg_bytes => 10000000});
+        case maps:get(queue_enabled, Opts, true) of
+            true -> replayq:open(#{dir => disk_queue_dir(Id), seg_bytes => 10000000});
             false -> undefined
         end,
     St = #{
         id => Id,
+        index => Index,
         batch_enabled => maps:get(batch_enabled, Opts, true),
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
-        cache_queue => Queue,
+        queue => Queue,
         acc => [],
         acc_left => BatchSize,
         tref => undefined
     },
-    {ok, do, St}.
+    {ok, blocked, St, {next_event, cast, resume}}.
 
-do(cast, {query, Request, ReplyFun}, #{batch_enabled := true} = State) ->
-    do_acc(ReplyFun, Request, State);
-do(cast, {query, Request, ReplyFun}, #{batch_enabled := false} = State) ->
-    do_query(ReplyFun, Request, State);
-do({call, From}, {query, Request}, #{batch_enabled := true} = State) ->
-    do_acc(From, Request, State);
-do({call, From}, {query, Request}, #{batch_enabled := false} = State) ->
-    do_query(From, Request, State);
-do(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) ->
-    {keep_state, flush(St#{tref := undefined})};
-do(info, {flush, _Ref}, _St) ->
+running(enter, _, _St) ->
     keep_state_and_data;
-do(info, Info, _St) ->
+running(cast, resume, _St) ->
+    keep_state_and_data;
+running(cast, block, St) ->
+    {next_state, block, St};
+running(cast, {query, Request, ReplyFun}, St) ->
+    query_or_acc(ReplyFun, Request, St);
+running({call, From}, {query, Request}, St) ->
+    query_or_acc(From, Request, St);
+running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) ->
+    {keep_state, flush(St#{tref := undefined})};
+running(info, {flush, _Ref}, _St) ->
+    keep_state_and_data;
+running(info, Info, _St) ->
     ?SLOG(error, #{msg => unexpected_msg, info => Info}),
     keep_state_and_data.
 
-do_acc(From, Request, #{acc := Acc, acc_left := Left} = St0) ->
+blocked(enter, _, _St) ->
+    keep_state_and_data;
+blocked(cast, block, _St) ->
+    keep_state_and_data;
+blocked(cast, resume, St) ->
+    do_resume(St);
+blocked(state_timeout, resume, St) ->
+    do_resume(St);
+blocked(cast, {query, Request, ReplyFun}, St) ->
+    handle_blocked(ReplyFun, Request, St);
+blocked({call, From}, {query, Request}, St) ->
+    handle_blocked(From, Request, St).
+
+terminate(_Reason, #{id := Id, index := Index}) ->
+    gproc_pool:disconnect_worker(Id, {Id, Index}).
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%==============================================================================
+pick(Id, Key) ->
+    Pid = gproc_pool:pick_worker(Id, Key),
+    case is_pid(Pid) of
+        true -> Pid;
+        false -> error({failed_to_pick_worker, {Id, Key}})
+    end.
+
+do_resume(#{queue := undefined} = St) ->
+    {next_state, running, St};
+do_resume(#{queue := Q, id := Id} = St) ->
+    case replayq:peek(Q) of
+        empty ->
+            {next_state, running, St, {state_timeout, ?RESUME_INTERVAL, resume}};
+        First ->
+            Result = call_query(Id, First),
+            case handle_query_result(Id, false, Result) of
+                %% Send failed because resource down
+                true ->
+                    {keep_state, St};
+                %% Send ok or failed but the resource is working
+                false ->
+                    %% We Send 'resume' to the end of the mailbox to give the worker
+                    %% a chance to process 'query' requests.
+                    {keep_state, St#{queue => drop_head(Q)}, {state_timeout, 0, resume}}
+            end
+    end.
+
+drop_head(Q) ->
+    {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
+    ok = replayq:ack(Q1, AckRef),
+    Q1.
+
+query_or_acc(From, Request, #{batch_enabled := true} = St) ->
+    acc_query(From, Request, St);
+query_or_acc(From, Request, #{batch_enabled := false} = St) ->
+    send_query(From, Request, St).
+
+acc_query(From, Request, #{acc := Acc, acc_left := Left} = St0) ->
     Acc1 = [?QUERY(From, Request) | Acc],
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
@@ -120,10 +190,19 @@ do_acc(From, Request, #{acc := Acc, acc_left := Left} = St0) ->
         false -> {keep_state, ensure_flush_timer(St)}
     end.
 
-do_query(From, Request, #{id := Id, cache_queue := Q0} = St0) ->
+send_query(From, Request, #{id := Id, queue := Q} = St) ->
     Result = call_query(Id, Request),
-    Q1 = reply_caller(Id, Q0, ?REPLY(From, Request, Result)),
-    {keep_state, St0#{cache_queue := Q1}}.
+    case reply_caller(Id, Q, ?REPLY(From, Request, Result)) of
+        true ->
+            {keep_state, St#{queue := maybe_append_queue(Q, [Request])}};
+        false ->
+            {next_state, blocked, St}
+    end.
+
+handle_blocked(From, Request, #{id := Id, queue := Q} = St) ->
+    Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
+    _ = reply_caller(Id, Q, ?REPLY(From, Request, Error)),
+    {keep_state, St#{queue := maybe_append_queue(Q, [Request])}}.
 
 flush(#{acc := []} = St) ->
     St;
@@ -132,101 +211,109 @@ flush(
         id := Id,
         acc := Batch,
         batch_size := Size,
-        cache_queue := Q0
+        queue := Q0
     } = St
 ) ->
-    BatchResults = call_batch_query(Id, Batch),
-    Q1 = batch_reply_caller(Id, Q0, BatchResults),
-    cancel_flush_timer(
-        St#{
-            acc_left := Size,
-            acc := [],
-            cache_queue := Q1
-        }
-    ).
+    BatchResults = maybe_expand_batch_result(call_batch_query(Id, Batch), Batch),
+    St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
+    case batch_reply_caller(Id, BatchResults) of
+        true ->
+            Q1 = maybe_append_queue(Q0, [Request || ?QUERY(_, Request) <- Batch]),
+            {keep_state, St1#{queue := Q1}};
+        false ->
+            {next_state, blocked, St1}
+    end.
 
-maybe_append_cache(undefined, _Request) -> undefined;
-maybe_append_cache(Q, Request) -> replayq:append(Q, Request).
+maybe_append_queue(undefined, _Query) -> undefined;
+maybe_append_queue(Q, Query) -> replayq:append(Q, Query).
 
-batch_reply_caller(Id, Q, BatchResults) ->
+batch_reply_caller(Id, BatchResults) ->
     lists:foldl(
-        fun(Reply, Q1) ->
-            reply_caller(Id, Q1, Reply)
+        fun(Reply, BlockWorker) ->
+            reply_caller(Id, BlockWorker, Reply)
         end,
-        Q,
+        false,
         BatchResults
     ).
 
-reply_caller(Id, Q, ?REPLY({ReplyFun, Args}, Request, Result)) when is_function(ReplyFun) ->
+reply_caller(Id, BlockWorker, ?REPLY(undefined, _, Result)) ->
+    handle_query_result(Id, BlockWorker, Result);
+reply_caller(Id, BlockWorker, ?REPLY({ReplyFun, Args}, _, Result)) ->
     ?SAFE_CALL(ReplyFun(Result, Args)),
-    handle_query_result(Id, Q, Request, Result);
-reply_caller(Id, Q, ?REPLY(From, Request, Result)) ->
+    handle_query_result(Id, BlockWorker, Result);
+reply_caller(Id, BlockWorker, ?REPLY(From, _, Result)) ->
     gen_statem:reply(From, Result),
-    handle_query_result(Id, Q, Request, Result).
+    handle_query_result(Id, BlockWorker, Result).
 
-handle_query_result(Id, Q, _Request, ok) ->
+handle_query_result(Id, BlockWorker, ok) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, success),
-    Q;
-handle_query_result(Id, Q, _Request, {ok, _}) ->
+    BlockWorker;
+handle_query_result(Id, BlockWorker, {ok, _}) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, success),
-    Q;
-handle_query_result(Id, Q, _Request, {error, _}) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
-    Q;
-handle_query_result(Id, Q, Request, {error, {resource_error, #{reason := not_connected}}}) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error),
-    maybe_append_cache(Q, Request);
-handle_query_result(Id, Q, _Request, {error, {resource_error, #{}}}) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error),
-    Q;
-handle_query_result(Id, Q, Request, {error, {exception, _}}) ->
+    BlockWorker;
+handle_query_result(Id, BlockWorker, ?RESOURCE_ERROR_M(exception, _)) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, exception),
-    maybe_append_cache(Q, Request).
+    BlockWorker;
+handle_query_result(_Id, _, ?RESOURCE_ERROR_M(NotWorking, _)) when
+    NotWorking == not_connected; NotWorking == blocked
+->
+    true;
+handle_query_result(_Id, BlockWorker, ?RESOURCE_ERROR_M(_, _)) ->
+    BlockWorker;
+handle_query_result(Id, BlockWorker, {error, _}) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
+    BlockWorker;
+handle_query_result(Id, _BlockWorker, {resource_down, _}) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down),
+    true.
 
 call_query(Id, Request) ->
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+    do_call_query(on_query, Id, Request).
+
+call_batch_query(Id, Batch) ->
+    do_call_query(on_batch_query, Id, Batch).
+
+do_call_query(Fun, Id, Data) ->
     case emqx_resource_manager:ets_lookup(Id) of
         {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
-            try Mod:on_query(Id, Request, ResourceState) of
+            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Data)),
+            try Mod:Fun(Id, Data, ResourceState) of
+                %% if the callback module (connector) wants to return an error that
+                %% makes the current resource goes into the `error` state, it should
+                %% return `{resource_down, Reason}`
                 Result -> Result
             catch
                 Err:Reason:ST ->
-                    ModB = atom_to_binary(Mod, utf8),
-                    Msg = <<"call failed, func: ", ModB/binary, ":on_query/3">>,
-                    exception_error(Reason, Msg, {Err, Reason, ST})
+                    Msg = io_lib:format(
+                        "call query failed, func: ~s:~s/3, error: ~0p",
+                        [Mod, Fun, {Err, Reason, ST}]
+                    ),
+                    ?RESOURCE_ERROR(exception, Msg)
             end;
         {ok, _Group, #{status := stopped}} ->
-            resource_error(stopped, <<"resource stopped or disabled">>);
+            ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
         {ok, _Group, _Data} ->
-            resource_error(not_connected, <<"resource not connected">>);
+            ?RESOURCE_ERROR(not_connected, "resource not connected");
         {error, not_found} ->
-            resource_error(not_found, <<"resource not found">>)
+            ?RESOURCE_ERROR(not_found, "resource not found")
     end.
 
-call_batch_query(Id, Batch) ->
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)),
-    case emqx_resource_manager:ets_lookup(Id) of
-        {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
-            try Mod:on_batch_query(Id, Batch, ResourceState) of
-                BatchResults -> BatchResults
-            catch
-                Err:Reason:ST ->
-                    ModB = atom_to_binary(Mod, utf8),
-                    Msg = <<"call failed, func: ", ModB/binary, ":on_batch_query/3">>,
-                    ?EXPAND(exception_error(Reason, Msg, {Err, Reason, ST}), Batch)
-            end;
-        {ok, _Group, _Data} ->
-            ?EXPAND(resource_error(not_connected, <<"resource not connected">>), Batch);
-        {error, not_found} ->
-            ?EXPAND(resource_error(not_found, <<"resource not found">>), Batch)
-    end.
+%% the result is already expaned by the `Mod:on_query/3`
+maybe_expand_batch_result(Results, _Batch) when is_list(Results) ->
+    Results;
+%% the `Mod:on_query/3` returns a sinle result for a batch, so it is need expand
+maybe_expand_batch_result(Result, Batch) ->
+    ?EXPAND(Result, Batch).
 
-resource_error(Reason, Msg) ->
-    {error, {resource_error, #{reason => Reason, msg => Msg}}}.
-exception_error(Reason, Msg, Details) ->
-    {error, {exception, #{reason => Reason, msg => Msg, details => Details}}}.
+%%==============================================================================
+
+-spec name(id(), integer()) -> atom().
+name(Id, Index) ->
+    list_to_atom(lists:concat([?MODULE, ":", Id, ":", Index])).
+
+disk_queue_dir(Id) ->
+    filename:join([emqx:data_dir(), Id, queue]).
 
-%% ==========================================
 ensure_flush_timer(St = #{tref := undefined, batch_time := T}) ->
     Ref = make_ref(),
     TRef = erlang:send_after(T, self(), {flush, Ref}),
@@ -239,14 +326,3 @@ cancel_flush_timer(St = #{tref := undefined}) ->
 cancel_flush_timer(St = #{tref := {TRef, _Ref}}) ->
     _ = erlang:cancel_timer(TRef),
     St#{tref => undefined}.
-
-query_mfa(InsertMode, Request, SyncTimeout) ->
-    {?MODULE, query_fun(InsertMode), query_args(InsertMode, Request, SyncTimeout)}.
-
-query_fun(<<"sync">>) -> query;
-query_fun(<<"async">>) -> query_async.
-
-query_args(<<"sync">>, Request, SyncTimeout) ->
-    [Request, SyncTimeout];
-query_args(<<"async">>, Request, _) ->
-    [Request].

From 0087b7c960a77c62f2caa873d425cafb22f0b866 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 27 Jul 2022 15:43:23 +0800
Subject: [PATCH 030/232] fix: remove the extra file replay.erl

---
 .../src/emqx_resource_manager.erl             |   1 +
 .../src/emqx_resource_worker.erl              |  25 +-
 apps/emqx_resource/src/replayq.erl            | 779 ------------------
 3 files changed, 20 insertions(+), 785 deletions(-)
 delete mode 100644 apps/emqx_resource/src/replayq.erl

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index ee18a6836..b8a3812b5 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -423,6 +423,7 @@ handle_remove_event(From, ClearMetrics, Data) ->
         true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id);
         false -> ok
     end,
+    ok = emqx_resource_sup:stop_workers(Data#data.id, Data#data.opts),
     {stop_and_reply, normal, [{reply, From, ok}]}.
 
 start_resource(Data, From) ->
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 3c5c7eefe..7acf2d0f9 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -27,7 +27,9 @@
 
 -export([
     start_link/3,
+    query/2,
     query/3,
+    query_async/3,
     query_async/4,
     block/1,
     resume/1
@@ -70,10 +72,18 @@ callback_mode() -> [state_functions, state_enter].
 start_link(Id, Index, Opts) ->
     gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []).
 
+-spec query(id(), request()) -> ok.
+query(Id, Request) ->
+    gen_statem:call(pick(Id, self()), {query, Request}).
+
 -spec query(id(), term(), request()) -> ok.
 query(Id, Key, Request) ->
     gen_statem:call(pick(Id, Key), {query, Request}).
 
+-spec query_async(id(), request(), reply_fun()) -> ok.
+query_async(Id, Request, ReplyFun) ->
+    gen_statem:cast(pick(Id, self()), {query, Request, ReplyFun}).
+
 -spec query_async(id(), term(), request(), reply_fun()) -> ok.
 query_async(Id, Key, Request, ReplyFun) ->
     gen_statem:cast(pick(Id, Key), {query, Request, ReplyFun}).
@@ -91,7 +101,7 @@ init({Id, Index, Opts}) ->
     BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE),
     Queue =
         case maps:get(queue_enabled, Opts, true) of
-            true -> replayq:open(#{dir => disk_queue_dir(Id), seg_bytes => 10000000});
+            true -> replayq:open(#{dir => disk_queue_dir(Id, Index), seg_bytes => 10000000});
             false -> undefined
         end,
     St = #{
@@ -157,13 +167,13 @@ do_resume(#{queue := undefined} = St) ->
 do_resume(#{queue := Q, id := Id} = St) ->
     case replayq:peek(Q) of
         empty ->
-            {next_state, running, St, {state_timeout, ?RESUME_INTERVAL, resume}};
+            {next_state, running, St};
         First ->
             Result = call_query(Id, First),
             case handle_query_result(Id, false, Result) of
                 %% Send failed because resource down
                 true ->
-                    {keep_state, St};
+                    {keep_state, St, {state_timeout, ?RESUME_INTERVAL, resume}};
                 %% Send ok or failed but the resource is working
                 false ->
                     %% We Send 'resume' to the end of the mailbox to give the worker
@@ -309,10 +319,13 @@ maybe_expand_batch_result(Result, Batch) ->
 
 -spec name(id(), integer()) -> atom().
 name(Id, Index) ->
-    list_to_atom(lists:concat([?MODULE, ":", Id, ":", Index])).
+    Mod = atom_to_list(?MODULE),
+    Id1 = binary_to_list(Id),
+    Index1 = integer_to_list(Index),
+    list_to_atom(lists:concat([Mod, ":", Id1, ":", Index1])).
 
-disk_queue_dir(Id) ->
-    filename:join([emqx:data_dir(), Id, queue]).
+disk_queue_dir(Id, Index) ->
+    filename:join([node(), emqx:data_dir(), Id, "queue:" ++ integer_to_list(Index)]).
 
 ensure_flush_timer(St = #{tref := undefined, batch_time := T}) ->
     Ref = make_ref(),
diff --git a/apps/emqx_resource/src/replayq.erl b/apps/emqx_resource/src/replayq.erl
deleted file mode 100644
index 437702a5e..000000000
--- a/apps/emqx_resource/src/replayq.erl
+++ /dev/null
@@ -1,779 +0,0 @@
--module(replayq).
-
--export([open/1, close/1]).
--export([append/2, pop/2, ack/2, ack_sync/2, peek/1, overflow/1]).
--export([count/1, bytes/1, is_empty/1, is_mem_only/1]).
-%% exported for troubleshooting
--export([do_read_items/2]).
-
-%% internal exports for beam reload
--export([committer_loop/2, default_sizer/1, default_marshaller/1]).
-
--export_type([config/0, q/0, ack_ref/0, sizer/0, marshaller/0]).
-
--define(NOTHING_TO_ACK, nothing_to_ack).
--define(PENDING_ACKS(Ref), {replayq_pending_acks, Ref}).
-
--type segno() :: pos_integer().
--type item() :: term().
--type count() :: non_neg_integer().
--type id() :: count().
--type bytes() :: non_neg_integer().
--type filename() :: file:filename_all().
--type dir() :: filename().
--type ack_ref() :: ?NOTHING_TO_ACK | {segno(), ID :: pos_integer()}.
--type sizer() :: fun((item()) -> bytes()).
--type marshaller() :: fun((item()) -> binary()).
-
--type config() :: #{
-    dir => dir(),
-    seg_bytes => bytes(),
-    mem_only => boolean(),
-    max_total_bytes => bytes(),
-    offload => boolean(),
-    sizer => sizer(),
-    marshaller => marshaller()
-}.
-%% writer cursor
--define(NO_FD, no_fd).
--type w_cur() :: #{
-    segno := segno(),
-    bytes := bytes(),
-    count := count(),
-    fd := ?NO_FD | file:fd()
-}.
-
--type stats() :: #{
-    bytes := bytes(),
-    count := count()
-}.
-
--opaque q() :: #{
-    config := mem_only | config(),
-    stats := stats(),
-    in_mem := queue:queue(in_mem_item()),
-    w_cur => w_cur(),
-    committer => pid(),
-    head_segno => segno(),
-    sizer := sizer(),
-    marshaller => marshaller(),
-    max_total_bytes := bytes()
-}.
-
--define(LAYOUT_VSN_0, 0).
--define(LAYOUT_VSN_1, 1).
--define(MAGIC, 841265288).
--define(SUFFIX, "replaylog").
--define(DEFAULT_POP_BYTES_LIMIT, 2000000).
--define(DEFAULT_POP_COUNT_LIMIT, 1000).
--define(DEFAULT_REPLAYQ_LIMIT, 2000000000).
--define(COMMIT(SEGNO, ID, From), {commit, SEGNO, ID, From}).
--define(NO_COMMIT_HIST, no_commit_hist).
--define(FIRST_SEGNO, 1).
--define(NEXT_SEGNO(N), (N + 1)).
--define(STOP, stop).
--define(MEM_ONLY_ITEM(Bytes, Item), {Bytes, Item}).
--define(DISK_CP_ITEM(Id, Bytes, Item), {Id, Bytes, Item}).
-
--type in_mem_item() ::
-    ?MEM_ONLY_ITEM(bytes(), item())
-    | ?DISK_CP_ITEM(id(), bytes(), item()).
-
--spec open(config()) -> q().
-open(#{mem_only := true} = C) ->
-    #{
-        stats => #{bytes => 0, count => 0},
-        in_mem => queue:new(),
-        sizer => get_sizer(C),
-        config => mem_only,
-        max_total_bytes => maps:get(max_total_bytes, C, ?DEFAULT_REPLAYQ_LIMIT)
-    };
-open(#{dir := Dir, seg_bytes := _} = Config) ->
-    ok = filelib:ensure_dir(filename:join(Dir, "foo")),
-    Sizer = get_sizer(Config),
-    Marshaller = get_marshaller(Config),
-    IsOffload = is_offload_mode(Config),
-    Q =
-        case delete_consumed_and_list_rest(Dir) of
-            [] ->
-                %% no old segments, start over from zero
-                #{
-                    stats => #{bytes => 0, count => 0},
-                    w_cur => init_writer(Dir, empty, IsOffload),
-                    committer => spawn_committer(?FIRST_SEGNO, Dir),
-                    head_segno => ?FIRST_SEGNO,
-                    in_mem => queue:new()
-                };
-            Segs ->
-                LastSegno = lists:last(Segs),
-                CommitHist = get_commit_hist(Dir),
-                Reader = fun(Seg, Ch) -> read_items(Dir, Seg, Ch, Sizer, Marshaller) end,
-                HeadItems = Reader(hd(Segs), CommitHist),
-                #{
-                    stats => collect_stats(HeadItems, tl(Segs), Reader),
-                    w_cur => init_writer(Dir, LastSegno, IsOffload),
-                    committer => spawn_committer(hd(Segs), Dir),
-                    head_segno => hd(Segs),
-                    in_mem => queue:from_list(HeadItems)
-                }
-        end,
-    Q#{
-        sizer => Sizer,
-        marshaller => Marshaller,
-        config => maps:without([sizer, marshaller], Config),
-        max_total_bytes => maps:get(max_total_bytes, Config, ?DEFAULT_REPLAYQ_LIMIT)
-    }.
-
--spec close(q() | w_cur()) -> ok | {error, any()}.
-close(#{config := mem_only}) ->
-    ok;
-close(#{w_cur := W_Cur, committer := Pid} = Q) ->
-    MRef = erlang:monitor(process, Pid),
-    Pid ! ?STOP,
-    unlink(Pid),
-    receive
-        {'DOWN', MRef, process, Pid, _Reason} ->
-            ok
-    end,
-    ok = maybe_dump_back_to_disk(Q),
-    do_close(W_Cur).
-
-do_close(#{fd := ?NO_FD}) -> ok;
-do_close(#{fd := Fd}) -> file:close(Fd).
-
-%% In case of offload mode, dump the unacked (and un-popped) on disk
-%% before close. this serves as a best-effort data loss protection
-maybe_dump_back_to_disk(#{config := Config} = Q) ->
-    case is_offload_mode(Config) of
-        true -> dump_back_to_disk(Q);
-        false -> ok
-    end.
-
-dump_back_to_disk(#{
-    config := #{dir := Dir},
-    head_segno := ReaderSegno,
-    in_mem := InMem,
-    marshaller := Marshaller
-}) ->
-    IoData0 = get_unacked(process_info(self(), dictionary), ReaderSegno, Marshaller),
-    Items1 = queue:to_list(InMem),
-    IoData1 = lists:map(fun(?DISK_CP_ITEM(_, _, I)) -> make_iodata(I, Marshaller) end, Items1),
-    %% ensure old segment file is deleted
-    ok = ensure_deleted(filename(Dir, ReaderSegno)),
-    %% rewrite the segment with what's currently in memory
-    IoData = [IoData0, IoData1],
-    case iolist_size(IoData) > 0 of
-        true ->
-            #{fd := Fd} = open_segment(Dir, ReaderSegno),
-            ok = file:write(Fd, [IoData0, IoData1]),
-            ok = file:close(Fd);
-        false ->
-            %% nothing to write
-            ok
-    end.
-
-get_unacked({dictionary, Dict}, ReaderSegno, Marshaller) ->
-    F = fun
-        ({?PENDING_ACKS(AckRef), Items}) ->
-            erase(?PENDING_ACKS(AckRef)),
-            {Segno, Id} = AckRef,
-            Segno =:= ReaderSegno andalso
-                {true, {Id, Items}};
-        (_) ->
-            false
-    end,
-    Pendings0 = lists:filtermap(F, Dict),
-    Pendings = lists:keysort(1, Pendings0),
-    do_get_unacked(Pendings, Marshaller).
-
-do_get_unacked([], _Marshaller) ->
-    [];
-do_get_unacked([{_, Items} | Rest], Marshaller) ->
-    [
-        [make_iodata(I, Marshaller) || I <- Items]
-        | do_get_unacked(Rest, Marshaller)
-    ].
-
--spec append(q(), [item()]) -> q().
-append(Q, []) ->
-    Q;
-append(
-    #{
-        config := mem_only,
-        in_mem := InMem,
-        stats := #{bytes := Bytes0, count := Count0},
-        sizer := Sizer
-    } = Q,
-    Items0
-) ->
-    {CountDiff, BytesDiff, Items} = transform(false, Items0, Sizer),
-
-    Stats = #{count => Count0 + CountDiff, bytes => Bytes0 + BytesDiff},
-    Q#{
-        stats := Stats,
-        in_mem := append_in_mem(Items, InMem)
-    };
-append(
-    #{
-        config := #{seg_bytes := BytesLimit, dir := Dir} = Config,
-        stats := #{bytes := Bytes0, count := Count0},
-        w_cur := #{count := CountInSeg, segno := WriterSegno} = W_Cur0,
-        head_segno := ReaderSegno,
-        sizer := Sizer,
-        marshaller := Marshaller,
-        in_mem := HeadItems0
-    } = Q,
-    Items0
-) ->
-    IoData = lists:map(fun(I) -> make_iodata(I, Marshaller) end, Items0),
-    {CountDiff, BytesDiff, Items} = transform(CountInSeg + 1, Items0, Sizer),
-    TotalBytes = Bytes0 + BytesDiff,
-    Stats = #{count => Count0 + CountDiff, bytes => TotalBytes},
-    IsOffload = is_offload_mode(Config),
-    W_Cur1 = do_append(W_Cur0, CountDiff, BytesDiff, IoData),
-    W_Cur =
-        case is_segment_full(W_Cur1, TotalBytes, BytesLimit, ReaderSegno, IsOffload) of
-            true ->
-                ok = do_close(W_Cur1),
-                %% get ready for the next append
-                open_segment(Dir, ?NEXT_SEGNO(WriterSegno));
-            false ->
-                W_Cur1
-        end,
-    HeadItems =
-        case ReaderSegno =:= WriterSegno of
-            true -> append_in_mem(Items, HeadItems0);
-            false -> HeadItems0
-        end,
-    Q#{
-        stats := Stats,
-        w_cur := W_Cur,
-        in_mem := HeadItems
-    }.
-
-%% @doc pop out at least one item from the queue.
-%% volume limited by `bytes_limit' and `count_limit'.
--spec pop(q(), #{bytes_limit => bytes(), count_limit => count()}) ->
-    {q(), ack_ref(), [item()]}.
-pop(Q, Opts) ->
-    Bytes = maps:get(bytes_limit, Opts, ?DEFAULT_POP_BYTES_LIMIT),
-    Count = maps:get(count_limit, Opts, ?DEFAULT_POP_COUNT_LIMIT),
-    true = (Count > 0),
-    pop(Q, Bytes, Count, ?NOTHING_TO_ACK, []).
-
-%% @doc peek the queue front item.
--spec peek(q()) -> empty | item().
-peek(#{in_mem := HeadItems}) ->
-    case queue:peek(HeadItems) of
-        empty -> empty;
-        {value, ?MEM_ONLY_ITEM(_, Item)} -> Item;
-        {value, ?DISK_CP_ITEM(_, _, Item)} -> Item
-    end.
-
-%% @doc Asynch-ly write the consumed item Segment number + ID to a file.
--spec ack(q(), ack_ref()) -> ok.
-ack(_, ?NOTHING_TO_ACK) ->
-    ok;
-ack(#{committer := Pid}, {Segno, Id} = AckRef) ->
-    _ = erlang:erase(?PENDING_ACKS(AckRef)),
-    Pid ! ?COMMIT(Segno, Id, false),
-    ok.
-
-%% @hidden Synced ack, for deterministic tests only
--spec ack_sync(q(), ack_ref()) -> ok.
-ack_sync(_, ?NOTHING_TO_ACK) ->
-    ok;
-ack_sync(#{committer := Pid}, {Segno, Id} = AckRef) ->
-    _ = erlang:erase(?PENDING_ACKS(AckRef)),
-    Ref = make_ref(),
-    Pid ! ?COMMIT(Segno, Id, {self(), Ref}),
-    receive
-        {Ref, ok} -> ok
-    end.
-
--spec count(q()) -> count().
-count(#{stats := #{count := Count}}) -> Count.
-
--spec bytes(q()) -> bytes().
-bytes(#{stats := #{bytes := Bytes}}) -> Bytes.
-
-is_empty(#{config := mem_only, in_mem := All}) ->
-    queue:is_empty(All);
-is_empty(
-    #{
-        w_cur := #{segno := WriterSegno},
-        head_segno := ReaderSegno,
-        in_mem := HeadItems
-    } = Q
-) ->
-    Result = ((WriterSegno =:= ReaderSegno) andalso queue:is_empty(HeadItems)),
-    %% assert
-    Result = (count(Q) =:= 0).
-
-%% @doc Returns number of bytes the size of the queue has exceeded
-%% total bytes limit. Result is negative when it is not overflow.
--spec overflow(q()) -> integer().
-overflow(#{
-    max_total_bytes := MaxTotalBytes,
-    stats := #{bytes := Bytes}
-}) ->
-    Bytes - MaxTotalBytes.
-
--spec is_mem_only(q()) -> boolean().
-is_mem_only(#{config := mem_only}) ->
-    true;
-is_mem_only(_) ->
-    false.
-
-%% internals =========================================================
-
-transform(Id, Items, Sizer) ->
-    transform(Id, Items, Sizer, 0, 0, []).
-
-transform(_Id, [], _Sizer, Count, Bytes, Acc) ->
-    {Count, Bytes, lists:reverse(Acc)};
-transform(Id, [Item0 | Rest], Sizer, Count, Bytes, Acc) ->
-    Size = Sizer(Item0),
-    {NextId, Item} =
-        case Id of
-            false -> {false, ?MEM_ONLY_ITEM(Size, Item0)};
-            N -> {N + 1, ?DISK_CP_ITEM(Id, Size, Item0)}
-        end,
-    transform(NextId, Rest, Sizer, Count + 1, Bytes + Size, [Item | Acc]).
-
-append_in_mem([], Q) -> Q;
-append_in_mem([Item | Rest], Q) -> append_in_mem(Rest, queue:in(Item, Q)).
-
-pop(Q, _Bytes, 0, AckRef, Acc) ->
-    Result = lists:reverse(Acc),
-    ok = maybe_save_pending_acks(AckRef, Q, Result),
-    {Q, AckRef, Result};
-pop(#{config := Cfg} = Q, Bytes, Count, AckRef, Acc) ->
-    case is_empty(Q) of
-        true ->
-            {Q, AckRef, lists:reverse(Acc)};
-        false when Cfg =:= mem_only ->
-            pop_mem(Q, Bytes, Count, Acc);
-        false ->
-            pop2(Q, Bytes, Count, AckRef, Acc)
-    end.
-
-pop_mem(
-    #{
-        in_mem := InMem,
-        stats := #{count := TotalCount, bytes := TotalBytes} = Stats
-    } = Q,
-    Bytes,
-    Count,
-    Acc
-) ->
-    case queue:out(InMem) of
-        {{value, ?MEM_ONLY_ITEM(Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] ->
-            {Q, ?NOTHING_TO_ACK, lists:reverse(Acc)};
-        {{value, ?MEM_ONLY_ITEM(Sz, Item)}, Rest} ->
-            NewQ = Q#{
-                in_mem := Rest,
-                stats := Stats#{
-                    count := TotalCount - 1,
-                    bytes := TotalBytes - Sz
-                }
-            },
-            pop(NewQ, Bytes - Sz, Count - 1, ?NOTHING_TO_ACK, [Item | Acc])
-    end.
-
-pop2(
-    #{
-        head_segno := ReaderSegno,
-        in_mem := HeadItems,
-        stats := #{count := TotalCount, bytes := TotalBytes} = Stats,
-        w_cur := #{segno := WriterSegno}
-    } = Q,
-    Bytes,
-    Count,
-    AckRef,
-    Acc
-) ->
-    case queue:out(HeadItems) of
-        {{value, ?DISK_CP_ITEM(_, Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] ->
-            %% taking the head item would cause exceeding size limit
-            {Q, AckRef, lists:reverse(Acc)};
-        {{value, ?DISK_CP_ITEM(Id, Sz, Item)}, Rest} ->
-            Q1 = Q#{
-                in_mem := Rest,
-                stats := Stats#{
-                    count := TotalCount - 1,
-                    bytes := TotalBytes - Sz
-                }
-            },
-            %% read the next segment in case current is drained
-            NewQ =
-                case queue:is_empty(Rest) andalso ReaderSegno < WriterSegno of
-                    true -> read_next_seg(Q1);
-                    false -> Q1
-                end,
-            NewAckRef = {ReaderSegno, Id},
-            pop(NewQ, Bytes - Sz, Count - 1, NewAckRef, [Item | Acc])
-    end.
-
-%% due to backward compatibility reasons for the ack api
-%% we ca nnot track pending acks in q() -- reason to use process dictionary
-maybe_save_pending_acks(?NOTHING_TO_ACK, _, _) ->
-    ok;
-maybe_save_pending_acks(AckRef, #{config := Config}, Items) ->
-    case is_offload_mode(Config) of
-        true ->
-            _ = erlang:put(?PENDING_ACKS(AckRef), Items),
-            ok;
-        false ->
-            ok
-    end.
-
-read_next_seg(
-    #{
-        config := #{dir := Dir} = Config,
-        head_segno := ReaderSegno,
-        w_cur := #{segno := WriterSegno, fd := Fd} = WCur0,
-        sizer := Sizer,
-        marshaller := Marshaller
-    } = Q
-) ->
-    NextSegno = ReaderSegno + 1,
-    %% reader has caught up to latest segment
-    case NextSegno =:= WriterSegno of
-        true ->
-            %% force flush to disk so the next read can get all bytes
-            ok = file:sync(Fd);
-        false ->
-            ok
-    end,
-    IsOffload = is_offload_mode(Config),
-    WCur =
-        case IsOffload andalso NextSegno =:= WriterSegno of
-            true ->
-                %% reader has caught up to latest segment in offload mode,
-                %% close the writer's fd. Continue in mem-only mode for the head segment
-                ok = do_close(WCur0),
-                WCur0#{fd := ?NO_FD};
-            false ->
-                WCur0
-        end,
-    NextSegItems = read_items(Dir, NextSegno, ?NO_COMMIT_HIST, Sizer, Marshaller),
-    Q#{
-        head_segno := NextSegno,
-        in_mem := queue:from_list(NextSegItems),
-        w_cur := WCur
-    }.
-
-delete_consumed_and_list_rest(Dir0) ->
-    Dir = unicode:characters_to_list(Dir0),
-    Segnos0 = lists:sort([parse_segno(N) || N <- filelib:wildcard("*." ?SUFFIX, Dir)]),
-    {SegnosToDelete, Segnos} = find_segnos_to_delete(Dir, Segnos0),
-    ok = lists:foreach(fun(Segno) -> ensure_deleted(filename(Dir, Segno)) end, SegnosToDelete),
-    case Segnos of
-        [] ->
-            %% delete commit file in case there is no segments left
-            %% segment number will start from 0 again.
-            ensure_deleted(commit_filename(Dir)),
-            [];
-        X ->
-            X
-    end.
-
-find_segnos_to_delete(Dir, Segnos) ->
-    CommitHist = get_commit_hist(Dir),
-    do_find_segnos_to_delete(Dir, Segnos, CommitHist).
-
-do_find_segnos_to_delete(_Dir, Segnos, ?NO_COMMIT_HIST) ->
-    {[], Segnos};
-do_find_segnos_to_delete(Dir, Segnos0, {CommittedSegno, CommittedId}) ->
-    {SegnosToDelete, Segnos} = lists:partition(fun(N) -> N < CommittedSegno end, Segnos0),
-    case
-        Segnos =/= [] andalso
-            hd(Segnos) =:= CommittedSegno andalso
-            is_all_consumed(Dir, CommittedSegno, CommittedId)
-    of
-        true ->
-            %% assert
-            CommittedSegno = hd(Segnos),
-            %% all items in the oldest segment have been consumed,
-            %% no need to keep this segment
-            {[CommittedSegno | SegnosToDelete], tl(Segnos)};
-        _ ->
-            {SegnosToDelete, Segnos}
-    end.
-
-%% ALL items are consumed if the committed item ID is no-less than the number
-%% of items in this segment
-is_all_consumed(Dir, CommittedSegno, CommittedId) ->
-    CommittedId >= erlang:length(do_read_items(Dir, CommittedSegno)).
-
-ensure_deleted(Filename) ->
-    case file:delete(Filename) of
-        ok -> ok;
-        {error, enoent} -> ok
-    end.
-
-%% The committer writes consumer's acked segmeng number + item ID
-%% to a file. The file is only read at start/restart.
-spawn_committer(ReaderSegno, Dir) ->
-    Name = iolist_to_binary(filename:join([Dir, committer])),
-    %% register a name to avoid having two committers spawned for the same dir
-    RegName = binary_to_atom(Name, utf8),
-    Pid = erlang:spawn_link(fun() -> committer_loop(ReaderSegno, Dir) end),
-    true = erlang:register(RegName, Pid),
-    Pid.
-
-committer_loop(ReaderSegno, Dir) ->
-    receive
-        ?COMMIT(Segno0, Id0, false) ->
-            {Segno, Id} = collect_async_commits(Segno0, Id0),
-            ok = handle_commit(ReaderSegno, Dir, Segno, Id, false),
-            ?MODULE:committer_loop(Segno, Dir);
-        ?COMMIT(Segno, Id, From) ->
-            ok = handle_commit(ReaderSegno, Dir, Segno, Id, From),
-            ?MODULE:committer_loop(Segno, Dir);
-        ?STOP ->
-            ok;
-        Msg ->
-            exit({replayq_committer_unkown_msg, Msg})
-    after 200 ->
-        ?MODULE:committer_loop(ReaderSegno, Dir)
-    end.
-
-handle_commit(ReaderSegno, Dir, Segno, Id, From) ->
-    IoData = io_lib:format("~p.\n", [#{segno => Segno, id => Id}]),
-    ok = do_commit(Dir, IoData),
-    case Segno > ReaderSegno of
-        true ->
-            SegnosToDelete = lists:seq(ReaderSegno, Segno - 1),
-            lists:foreach(fun(N) -> ok = ensure_deleted(filename(Dir, N)) end, SegnosToDelete);
-        false ->
-            ok
-    end,
-    ok = reply_ack_ok(From).
-
-%% Collect async acks which are already sent in the mailbox,
-%% and keep only the last one for the current segment.
-collect_async_commits(Segno, Id) ->
-    receive
-        ?COMMIT(Segno, AnotherId, false) ->
-            collect_async_commits(Segno, AnotherId)
-    after 0 ->
-        {Segno, Id}
-    end.
-
-reply_ack_ok({Pid, Ref}) ->
-    Pid ! {Ref, ok},
-    ok;
-reply_ack_ok(_) ->
-    ok.
-
-get_commit_hist(Dir) ->
-    CommitFile = commit_filename(Dir),
-    case filelib:is_regular(CommitFile) of
-        true ->
-            {ok, [#{segno := Segno, id := Id}]} = file:consult(CommitFile),
-            {Segno, Id};
-        false ->
-            ?NO_COMMIT_HIST
-    end.
-
-do_commit(Dir, IoData) ->
-    TmpName = commit_filename(Dir, "COMMIT.tmp"),
-    Name = commit_filename(Dir),
-    ok = file:write_file(TmpName, IoData),
-    ok = file:rename(TmpName, Name).
-
-commit_filename(Dir) ->
-    commit_filename(Dir, "COMMIT").
-
-commit_filename(Dir, Name) ->
-    filename:join([Dir, Name]).
-
-do_append(
-    #{fd := ?NO_FD, bytes := Bytes0, count := Count0} = Cur,
-    Count,
-    Bytes,
-    _IoData
-) ->
-    %% offload mode, fd is not initialized yet
-    Cur#{
-        bytes => Bytes0 + Bytes,
-        count => Count0 + Count
-    };
-do_append(
-    #{fd := Fd, bytes := Bytes0, count := Count0} = Cur,
-    Count,
-    Bytes,
-    IoData
-) ->
-    ok = file:write(Fd, IoData),
-    Cur#{
-        bytes => Bytes0 + Bytes,
-        count => Count0 + Count
-    }.
-
-read_items(Dir, Segno, CommitHist, Sizer, Marshaller) ->
-    Items0 = do_read_items(Dir, Segno),
-    Items =
-        case CommitHist of
-            ?NO_COMMIT_HIST ->
-                %% no commit hist, return all
-                Items0;
-            {CommitedSegno, _} when CommitedSegno < Segno ->
-                %% committed at an older segment
-                Items0;
-            {Segno, CommittedId} ->
-                %% committed at current segment keep only the tail
-                {_, R} = lists:splitwith(fun({I, _}) -> I =< CommittedId end, Items0),
-                R
-        end,
-    lists:map(
-        fun({Id, Bin}) ->
-            Item = Marshaller(Bin),
-            Size = Sizer(Item),
-            ?DISK_CP_ITEM(Id, Size, Item)
-        end,
-        Items
-    ).
-
-do_read_items(Dir, Segno) ->
-    Filename = filename(Dir, Segno),
-    {ok, Bin} = file:read_file(Filename),
-    case parse_items(Bin, 1, []) of
-        {Items, <<>>} ->
-            Items;
-        {Items, Corrupted} ->
-            error_logger:error_msg(
-                "corrupted replayq log: ~s, skipped ~p bytes",
-                [Filename, size(Corrupted)]
-            ),
-            Items
-    end.
-
-parse_items(<<>>, _Id, Acc) ->
-    {lists:reverse(Acc), <<>>};
-parse_items(
-    <<?LAYOUT_VSN_1:8, ?MAGIC:32/unsigned-integer, CRC:32/unsigned-integer,
-        Size:32/unsigned-integer, Item:Size/binary, Rest/binary>> = All,
-    Id,
-    Acc
-) ->
-    case CRC =:= erlang:crc32(Item) of
-        true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]);
-        false -> {lists:reverse(Acc), All}
-    end;
-parse_items(
-    <<?LAYOUT_VSN_0:8, CRC:32/unsigned-integer, Size:32/unsigned-integer, Item:Size/binary,
-        Rest/binary>> = All,
-    Id,
-    Acc
-) ->
-    case CRC =:= erlang:crc32(Item) andalso Item =/= <<>> of
-        true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]);
-        false -> {lists:reverse(Acc), All}
-    end;
-parse_items(Corrupted, _Id, Acc) ->
-    {lists:reverse(Acc), Corrupted}.
-
-make_iodata(Item0, Marshaller) ->
-    Item = Marshaller(Item0),
-    Size = size(Item),
-    CRC = erlang:crc32(Item),
-    [
-        <<?LAYOUT_VSN_1:8, ?MAGIC:32/unsigned-integer, CRC:32/unsigned-integer,
-            Size:32/unsigned-integer>>,
-        Item
-    ].
-
-collect_stats(HeadItems, SegsOnDisk, Reader) ->
-    ItemF = fun(?DISK_CP_ITEM(_Id, Sz, _Item), {B, C}) ->
-        {B + Sz, C + 1}
-    end,
-    Acc0 = lists:foldl(ItemF, {0, 0}, HeadItems),
-    {Bytes, Count} =
-        lists:foldl(
-            fun(Segno, Acc) ->
-                Items = Reader(Segno, ?NO_COMMIT_HIST),
-                lists:foldl(ItemF, Acc, Items)
-            end,
-            Acc0,
-            SegsOnDisk
-        ),
-    #{bytes => Bytes, count => Count}.
-
-parse_segno(Filename) ->
-    [Segno, ?SUFFIX] = string:tokens(Filename, "."),
-    list_to_integer(Segno).
-
-filename(Dir, Segno) ->
-    Name = lists:flatten(io_lib:format("~10.10.0w." ?SUFFIX, [Segno])),
-    filename:join(Dir, Name).
-
-%% open the current segment for write if it is empty
-%% otherwise rollout to the next segment
--spec init_writer(dir(), empty | segno(), boolean()) -> w_cur().
-init_writer(_Dir, empty, true) ->
-    %% clean start for offload mode
-    #{fd => ?NO_FD, segno => ?FIRST_SEGNO, bytes => 0, count => 0};
-init_writer(Dir, empty, false) ->
-    open_segment(Dir, ?FIRST_SEGNO);
-init_writer(Dir, Segno, _IsOffload) when is_number(Segno) ->
-    Filename = filename(Dir, Segno),
-    case filelib:file_size(Filename) of
-        0 -> open_segment(Dir, Segno);
-        _ -> open_segment(Dir, ?NEXT_SEGNO(Segno))
-    end.
-
--spec open_segment(dir(), segno()) -> w_cur().
-open_segment(Dir, Segno) ->
-    Filename = filename(Dir, Segno),
-    %% raw so there is no need to go through the single gen_server file_server
-    {ok, Fd} = file:open(Filename, [raw, read, write, binary, delayed_write]),
-    #{fd => Fd, segno => Segno, bytes => 0, count => 0}.
-
-get_sizer(C) ->
-    maps:get(sizer, C, fun ?MODULE:default_sizer/1).
-
-get_marshaller(C) ->
-    maps:get(marshaller, C, fun ?MODULE:default_marshaller/1).
-
-is_offload_mode(Config) when is_map(Config) ->
-    maps:get(offload, Config, false).
-
-default_sizer(I) when is_binary(I) -> erlang:size(I).
-
-default_marshaller(I) when is_binary(I) -> I.
-
-is_segment_full(
-    #{segno := WriterSegno, bytes := SegmentBytes},
-    TotalBytes,
-    SegmentBytesLimit,
-    ReaderSegno,
-    true
-) ->
-    %% in offload mode, when reader is lagging behind, we try
-    %% writer rolls to a new segment when file size limit is reached
-    %% when reader is reading off from the same segment as writer
-    %% i.e. the in memory queue, only start writing to segment file
-    %% when total bytes (in memory) is reached segment limit
-    %%
-    %% NOTE: we never shrink segment bytes, even when popping out
-    %% from the in-memory queue.
-    case ReaderSegno < WriterSegno of
-        true -> SegmentBytes >= SegmentBytesLimit;
-        false -> TotalBytes >= SegmentBytesLimit
-    end;
-is_segment_full(
-    #{bytes := SegmentBytes},
-    _TotalBytes,
-    SegmentBytesLimit,
-    _ReaderSegno,
-    false
-) ->
-    %% here we check if segment size is greater than segment size limit
-    %% after append based on the assumption that the items are usually
-    %% very small in size comparing to segment size.
-    %% We can change implementation to split items list to avoid
-    %% segment overflow if really necessary
-    SegmentBytes >= SegmentBytesLimit.

From 2fb42e4d37442b141e1b0259bc71ae5807502704 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 28 Jul 2022 17:40:04 +0800
Subject: [PATCH 031/232] refactor: create emqx_resource_worker_sup for
 resource workers

---
 .../emqx_authn_jwks_connector.erl             |  12 +-
 .../src/simple_authn/emqx_authn_mongodb.erl   |   2 +-
 .../src/emqx_connector_http.erl               |  29 ++--
 .../src/emqx_connector_ldap.erl               |   9 +-
 .../src/emqx_connector_mongo.erl              |  10 +-
 .../src/emqx_connector_mqtt.erl               |  10 +-
 .../src/emqx_connector_mysql.erl              |  17 +--
 .../src/emqx_connector_pgsql.erl              |  13 +-
 .../src/emqx_connector_redis.erl              |   9 +-
 apps/emqx_resource/include/emqx_resource.hrl  |  14 +-
 apps/emqx_resource/src/emqx_resource.erl      |  32 +----
 .../src/emqx_resource_manager.erl             |  15 +-
 apps/emqx_resource/src/emqx_resource_sup.erl  |  85 ++---------
 .../src/emqx_resource_worker.erl              | 132 ++++++++++-------
 .../src/emqx_resource_worker_sup.erl          | 136 ++++++++++++++++++
 .../test/emqx_resource_SUITE.erl              |  77 +++++-----
 .../emqx_resource/test/emqx_test_resource.erl |  27 ++--
 17 files changed, 332 insertions(+), 297 deletions(-)
 create mode 100644 apps/emqx_resource/src/emqx_resource_worker_sup.erl

diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl
index 8f98e2f1e..cd8451ac9 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl
@@ -24,7 +24,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2,
     connect/1
 ]).
@@ -45,7 +45,7 @@ on_start(InstId, Opts) ->
 on_stop(_InstId, #{pool_name := PoolName}) ->
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
-on_query(InstId, get_jwks, AfterQuery, #{pool_name := PoolName}) ->
+on_query(InstId, get_jwks, #{pool_name := PoolName}) ->
     Result = ecpool:pick_and_do(PoolName, {emqx_authn_jwks_client, get_jwks, []}, no_handover),
     case Result of
         {error, Reason} ->
@@ -54,20 +54,18 @@ on_query(InstId, get_jwks, AfterQuery, #{pool_name := PoolName}) ->
                 connector => InstId,
                 command => get_jwks,
                 reason => Reason
-            }),
-            emqx_resource:query_failed(AfterQuery);
+            });
         _ ->
-            emqx_resource:query_success(AfterQuery)
+            ok
     end,
     Result;
-on_query(_InstId, {update, Opts}, AfterQuery, #{pool_name := PoolName}) ->
+on_query(_InstId, {update, Opts}, #{pool_name := PoolName}) ->
     lists:foreach(
         fun({_, Worker}) ->
             ok = ecpool_worker:exec(Worker, {emqx_authn_jwks_client, update, [Opts]}, infinity)
         end,
         ecpool:workers(PoolName)
     ),
-    emqx_resource:query_success(AfterQuery),
     ok.
 
 on_get_status(_InstId, #{pool_name := PoolName}) ->
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
index f7249ae57..ff9c97717 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
@@ -174,7 +174,7 @@ authenticate(
                 reason => Reason
             }),
             ignore;
-        Doc ->
+        {ok, Doc} ->
             case check_password(Password, Doc, State) of
                 ok ->
                     {ok, is_superuser(Doc, State)};
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 59b4ddffa..e0d5ccfe0 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -28,7 +28,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -225,7 +225,7 @@ on_stop(InstId, #{pool_name := PoolName}) ->
     }),
     ehttpc_sup:stop_pool(PoolName).
 
-on_query(InstId, {send_message, Msg}, AfterQuery, State) ->
+on_query(InstId, {send_message, Msg}, State) ->
     case maps:get(request, State, undefined) of
         undefined ->
             ?SLOG(error, #{msg => "request_not_found", connector => InstId});
@@ -241,18 +241,16 @@ on_query(InstId, {send_message, Msg}, AfterQuery, State) ->
             on_query(
                 InstId,
                 {undefined, Method, {Path, Headers, Body}, Timeout, Retry},
-                AfterQuery,
                 State
             )
     end;
-on_query(InstId, {Method, Request}, AfterQuery, State) ->
-    on_query(InstId, {undefined, Method, Request, 5000, 2}, AfterQuery, State);
-on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
-    on_query(InstId, {undefined, Method, Request, Timeout, 2}, AfterQuery, State);
+on_query(InstId, {Method, Request}, State) ->
+    on_query(InstId, {undefined, Method, Request, 5000, 2}, State);
+on_query(InstId, {Method, Request, Timeout}, State) ->
+    on_query(InstId, {undefined, Method, Request, Timeout, 2}, State);
 on_query(
     InstId,
     {KeyOrNum, Method, Request, Timeout, Retry},
-    AfterQuery,
     #{pool_name := PoolName, base_path := BasePath} = State
 ) ->
     ?TRACE(
@@ -275,32 +273,29 @@ on_query(
     of
         {error, Reason} ->
             ?SLOG(error, #{
-                msg => "http_connector_do_reqeust_failed",
+                msg => "http_connector_do_request_failed",
                 request => NRequest,
                 reason => Reason,
                 connector => InstId
-            }),
-            emqx_resource:query_failed(AfterQuery);
+            });
         {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
-            emqx_resource:query_success(AfterQuery);
+            ok;
         {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
-            emqx_resource:query_success(AfterQuery);
+            ok;
         {ok, StatusCode, _} ->
             ?SLOG(error, #{
                 msg => "http connector do request, received error response",
                 request => NRequest,
                 connector => InstId,
                 status_code => StatusCode
-            }),
-            emqx_resource:query_failed(AfterQuery);
+            });
         {ok, StatusCode, _, _} ->
             ?SLOG(error, #{
                 msg => "http connector do request, received error response",
                 request => NRequest,
                 connector => InstId,
                 status_code => StatusCode
-            }),
-            emqx_resource:query_failed(AfterQuery)
+            })
     end,
     Result.
 
diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl
index 195aa89a9..51d18b534 100644
--- a/apps/emqx_connector/src/emqx_connector_ldap.erl
+++ b/apps/emqx_connector/src/emqx_connector_ldap.erl
@@ -27,7 +27,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -99,7 +99,7 @@ on_stop(InstId, #{poolname := PoolName}) ->
     }),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
-on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
+on_query(InstId, {search, Base, Filter, Attributes}, #{poolname := PoolName} = State) ->
     Request = {Base, Filter, Attributes},
     ?TRACE(
         "QUERY",
@@ -119,10 +119,9 @@ on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := P
                 request => Request,
                 connector => InstId,
                 reason => Reason
-            }),
-            emqx_resource:query_failed(AfterQuery);
+            });
         _ ->
-            emqx_resource:query_success(AfterQuery)
+            ok
     end,
     Result.
 
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index 5b07c5003..db8b1e632 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -27,7 +27,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -189,7 +189,6 @@ on_stop(InstId, #{poolname := PoolName}) ->
 on_query(
     InstId,
     {Action, Collection, Filter, Projector},
-    AfterQuery,
     #{poolname := PoolName} = State
 ) ->
     Request = {Action, Collection, Filter, Projector},
@@ -212,14 +211,11 @@ on_query(
                 reason => Reason,
                 connector => InstId
             }),
-            emqx_resource:query_failed(AfterQuery),
             {error, Reason};
         {ok, Cursor} when is_pid(Cursor) ->
-            emqx_resource:query_success(AfterQuery),
-            mc_cursor:foldl(fun(O, Acc2) -> [O | Acc2] end, [], Cursor, 1000);
+            {ok, mc_cursor:foldl(fun(O, Acc2) -> [O | Acc2] end, [], Cursor, 1000)};
         Result ->
-            emqx_resource:query_success(AfterQuery),
-            Result
+            {ok, Result}
     end.
 
 -dialyzer({nowarn_function, [on_get_status/2]}).
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 21e201504..98635de3f 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -37,7 +37,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -181,12 +181,12 @@ on_stop(_InstId, #{name := InstanceId}) ->
             })
     end.
 
-on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) ->
-    emqx_resource:query_success(AfterQuery);
-on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) ->
+on_query(_InstId, {message_received, _Msg}, _State) ->
+    ok;
+on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) ->
     ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
     emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
-    emqx_resource:query_success(AfterQuery).
+    ok.
 
 on_get_status(_InstId, #{name := InstanceId, bridge_conf := Conf}) ->
     AutoReconn = maps:get(auto_reconnect, Conf, true),
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index 409da4060..e818bc6ef 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -26,7 +26,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -122,14 +122,13 @@ on_stop(InstId, #{poolname := PoolName}) ->
     }),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
-on_query(InstId, {TypeOrKey, SQLOrKey}, AfterQuery, State) ->
-    on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, AfterQuery, State);
-on_query(InstId, {TypeOrKey, SQLOrKey, Params}, AfterQuery, State) ->
-    on_query(InstId, {TypeOrKey, SQLOrKey, Params, default_timeout}, AfterQuery, State);
+on_query(InstId, {TypeOrKey, SQLOrKey}, State) ->
+    on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, State);
+on_query(InstId, {TypeOrKey, SQLOrKey, Params}, State) ->
+    on_query(InstId, {TypeOrKey, SQLOrKey, Params, default_timeout}, State);
 on_query(
     InstId,
     {TypeOrKey, SQLOrKey, Params, Timeout},
-    AfterQuery,
     #{poolname := PoolName, prepare_statement := Prepares} = State
 ) ->
     LogMeta = #{connector => InstId, sql => SQLOrKey, state => State},
@@ -147,7 +146,6 @@ on_query(
             ),
             %% kill the poll worker to trigger reconnection
             _ = exit(Conn, restart),
-            emqx_resource:query_failed(AfterQuery),
             Result;
         {error, not_prepared} ->
             ?SLOG(
@@ -157,13 +155,12 @@ on_query(
             case prepare_sql(Prepares, PoolName) of
                 ok ->
                     %% not return result, next loop will try again
-                    on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, AfterQuery, State);
+                    on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, State);
                 {error, Reason} ->
                     ?SLOG(
                         error,
                         LogMeta#{msg => "mysql_connector_do_prepare_failed", reason => Reason}
                     ),
-                    emqx_resource:query_failed(AfterQuery),
                     {error, Reason}
             end;
         {error, Reason} ->
@@ -171,10 +168,8 @@ on_query(
                 error,
                 LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
             ),
-            emqx_resource:query_failed(AfterQuery),
             Result;
         _ ->
-            emqx_resource:query_success(AfterQuery),
             Result
     end.
 
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index 6f89e7ff1..d31c1316f 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -29,7 +29,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -116,9 +116,9 @@ on_stop(InstId, #{poolname := PoolName}) ->
     }),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
-on_query(InstId, {Type, NameOrSQL}, AfterQuery, #{poolname := _PoolName} = State) ->
-    on_query(InstId, {Type, NameOrSQL, []}, AfterQuery, State);
-on_query(InstId, {Type, NameOrSQL, Params}, AfterQuery, #{poolname := PoolName} = State) ->
+on_query(InstId, {Type, NameOrSQL}, #{poolname := _PoolName} = State) ->
+    on_query(InstId, {Type, NameOrSQL, []}, State);
+on_query(InstId, {Type, NameOrSQL, Params}, #{poolname := PoolName} = State) ->
     ?SLOG(debug, #{
         msg => "postgresql connector received sql query",
         connector => InstId,
@@ -132,10 +132,9 @@ on_query(InstId, {Type, NameOrSQL, Params}, AfterQuery, #{poolname := PoolName}
                 connector => InstId,
                 sql => NameOrSQL,
                 reason => Reason
-            }),
-            emqx_resource:query_failed(AfterQuery);
+            });
         _ ->
-            emqx_resource:query_success(AfterQuery)
+            ok
     end,
     Result.
 
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 67310dbac..4826a170b 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -28,7 +28,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -177,7 +177,7 @@ on_stop(InstId, #{poolname := PoolName, type := Type}) ->
         _ -> emqx_plugin_libs_pool:stop_pool(PoolName)
     end.
 
-on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
+on_query(InstId, {cmd, Command}, #{poolname := PoolName, type := Type} = State) ->
     ?TRACE(
         "QUERY",
         "redis_connector_received",
@@ -195,10 +195,9 @@ on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := T
                 connector => InstId,
                 sql => Command,
                 reason => Reason
-            }),
-            emqx_resource:query_failed(AfterCommand);
+            });
         _ ->
-            emqx_resource:query_success(AfterCommand)
+            ok
     end,
     Result.
 
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index d6f959510..13ffff587 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -21,7 +21,7 @@
 -type resource_config() :: term().
 -type resource_spec() :: map().
 -type resource_state() :: term().
--type resource_status() :: connected | disconnected | connecting.
+-type resource_status() :: connected | disconnected | connecting | stopped.
 -type resource_data() :: #{
     id := resource_id(),
     mod := module(),
@@ -45,13 +45,11 @@
     %% periodically.
     auto_retry_interval => integer()
 }.
--type after_query() ::
-    {[OnSuccess :: after_query_fun()], [OnFailed :: after_query_fun()]}
-    | undefined.
-
-%% the `after_query_fun()` is mainly for callbacks that increment counters or do some fallback
-%% actions upon query failure
--type after_query_fun() :: {fun((...) -> ok), Args :: [term()]}.
+-type query_result() ::
+    ok
+    | {ok, term()}
+    | {error, term()}
+    | {resource_down, term()}.
 
 -define(TEST_ID_PREFIX, "_test_:").
 -define(RES_METRICS, resource_metrics).
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 793b9f446..081264315 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -23,13 +23,6 @@
 
 -export([list_types/0]).
 
-%% APIs for behaviour implementations
-
--export([
-    query_success/1,
-    query_failed/1
-]).
-
 %% APIs for instances
 
 -export([
@@ -113,7 +106,8 @@
 -export([inc_metrics_funcs/1, inc_success/1, inc_failed/1]).
 
 -optional_callbacks([
-    on_query/4,
+    on_query/3,
+    on_batch_query/3,
     on_get_status/2
 ]).
 
@@ -125,7 +119,9 @@
 -callback on_stop(resource_id(), resource_state()) -> term().
 
 %% when calling emqx_resource:query/3
--callback on_query(resource_id(), Request :: term(), after_query(), resource_state()) -> term().
+-callback on_query(resource_id(), Request :: term(), resource_state()) -> query_result().
+
+-callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result().
 
 %% when calling emqx_resource:health_check/2
 -callback on_get_status(resource_id(), resource_state()) ->
@@ -149,22 +145,6 @@ is_resource_mod(Module) ->
             proplists:get_value(behaviour, Info, []),
     lists:member(?MODULE, Behaviour).
 
--spec query_success(after_query()) -> ok.
-query_success(undefined) -> ok;
-query_success({OnSucc, _}) -> exec_query_after_calls(OnSucc).
-
--spec query_failed(after_query()) -> ok.
-query_failed(undefined) -> ok;
-query_failed({_, OnFailed}) -> exec_query_after_calls(OnFailed).
-
-exec_query_after_calls(Funcs) ->
-    lists:foreach(
-        fun({Fun, Arg}) ->
-            emqx_resource_utils:safe_exec(Fun, Arg)
-        end,
-        Funcs
-    ).
-
 %% =================================================================================
 %% APIs for resource instances
 %% =================================================================================
@@ -247,7 +227,7 @@ query(ResId, Request) ->
     emqx_resource_worker:query(ResId, Request).
 
 -spec query_async(resource_id(), Request :: term(), emqx_resource_worker:reply_fun()) ->
-    ok.
+    Result :: term().
 query_async(ResId, Request, ReplyFun) ->
     emqx_resource_worker:query_async(ResId, Request, ReplyFun).
 
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index b8a3812b5..b5bcbd330 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -53,7 +53,7 @@
 
 -define(SHORT_HEALTHCHECK_INTERVAL, 1000).
 -define(HEALTHCHECK_INTERVAL, 15000).
--define(ETS_TABLE, emqx_resource_manager).
+-define(ETS_TABLE, ?MODULE).
 -define(WAIT_FOR_RESOURCE_DELAY, 100).
 -define(T_OPERATION, 5000).
 -define(T_LOOKUP, 1000).
@@ -114,9 +114,9 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         [matched, success, failed, exception, resource_down],
         [matched]
     ),
+    ok = emqx_resource_worker_sup:start_workers(ResId, Opts),
     case maps:get(start_after_created, Opts, true) of
         true ->
-            ok = emqx_resource_sup:start_workers(ResId, Opts),
             wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000));
         false ->
             ok
@@ -317,7 +317,7 @@ handle_event({call, From}, health_check, _State, Data) ->
     handle_manually_health_check(From, Data);
 % State: CONNECTING
 handle_event(enter, _OldState, connecting, Data) ->
-    UpdatedData = Data#data{status = connected},
+    UpdatedData = Data#data{status = connecting},
     insert_cache(Data#data.id, Data#data.group, Data),
     Actions = [{state_timeout, 0, health_check}],
     {keep_state, UpdatedData, Actions};
@@ -332,7 +332,7 @@ handle_event(enter, _OldState, connected, Data) ->
     UpdatedData = Data#data{status = connected},
     insert_cache(Data#data.id, Data#data.group, UpdatedData),
     _ = emqx_alarm:deactivate(Data#data.id),
-    Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}],
+    Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}],
     {next_state, connected, UpdatedData, Actions};
 handle_event(state_timeout, health_check, connected, Data) ->
     handle_connected_health_check(Data);
@@ -423,7 +423,7 @@ handle_remove_event(From, ClearMetrics, Data) ->
         true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id);
         false -> ok
     end,
-    ok = emqx_resource_sup:stop_workers(Data#data.id, Data#data.opts),
+    ok = emqx_resource_worker_sup:stop_workers(Data#data.id, Data#data.opts),
     {stop_and_reply, normal, [{reply, From, ok}]}.
 
 start_resource(Data, From) ->
@@ -487,7 +487,7 @@ handle_connected_health_check(Data) ->
         Data,
         fun
             (connected, UpdatedData) ->
-                Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}],
+                Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}],
                 {keep_state, UpdatedData, Actions};
             (Status, UpdatedData) ->
                 ?SLOG(error, #{
@@ -510,6 +510,9 @@ with_health_check(Data, Func) ->
     insert_cache(ResId, UpdatedData#data.group, UpdatedData),
     Func(Status, UpdatedData).
 
+health_check_interval(Opts) ->
+    maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL).
+
 maybe_alarm(connected, _ResId) ->
     ok;
 maybe_alarm(_Status, <<?TEST_ID_PREFIX, _/binary>>) ->
diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl
index 14db50d01..920743101 100644
--- a/apps/emqx_resource/src/emqx_resource_sup.erl
+++ b/apps/emqx_resource/src/emqx_resource_sup.erl
@@ -19,7 +19,7 @@
 
 -behaviour(supervisor).
 
--export([start_link/0, start_workers/2, stop_workers/2]).
+-export([start_link/0]).
 
 -export([init/1]).
 
@@ -29,7 +29,6 @@ start_link() ->
 init([]) ->
     SupFlags = #{strategy => one_for_one, intensity => 10, period => 10},
     Metrics = emqx_metrics_worker:child_spec(?RES_METRICS),
-
     ResourceManager =
         #{
             id => emqx_resource_manager_sup,
@@ -39,79 +38,11 @@ init([]) ->
             type => supervisor,
             modules => [emqx_resource_manager_sup]
         },
-    {ok, {SupFlags, [Metrics, ResourceManager]}}.
-
-start_workers(ResId, Opts) ->
-    PoolSize = pool_size(Opts),
-    _ = ensure_worker_pool(ResId, hash, [{size, PoolSize}]),
-    lists:foreach(
-        fun(Idx) ->
-            _ = ensure_worker_added(ResId, {ResId, Idx}, Idx),
-            ok = ensure_worker_started(ResId, Idx, Opts)
-        end,
-        lists:seq(1, PoolSize)
-    ).
-
-stop_workers(ResId, Opts) ->
-    PoolSize = pool_size(Opts),
-    lists:foreach(
-        fun(Idx) ->
-            ok = ensure_worker_stopped(ResId, Idx),
-            ok = ensure_worker_removed(ResId, {ResId, Idx})
-        end,
-        lists:seq(1, PoolSize)
-    ),
-    _ = gproc_pool:delete(ResId),
-    ok.
-
-pool_size(Opts) ->
-    maps:get(worker_pool_size, Opts, erlang:system_info(schedulers_online)).
-
-ensure_worker_pool(Pool, Type, Opts) ->
-    try
-        gproc_pool:new(Pool, Type, Opts)
-    catch
-        error:exists -> ok
-    end,
-    ok.
-
-ensure_worker_added(Pool, Name, Slot) ->
-    try
-        gproc_pool:add_worker(Pool, Name, Slot)
-    catch
-        error:exists -> ok
-    end,
-    ok.
-
-ensure_worker_removed(Pool, Name) ->
-    _ = gproc_pool:remove_worker(Pool, Name),
-    ok.
-
--define(CHILD_ID(MOD, RESID, INDEX), {MOD, RESID, INDEX}).
-ensure_worker_started(ResId, Idx, Opts) ->
-    Mod = emqx_resource_worker,
-    Spec = #{
-        id => ?CHILD_ID(Mod, ResId, Idx),
-        start => {Mod, start_link, [ResId, Idx, Opts]},
-        restart => transient,
-        shutdown => 5000,
-        type => worker,
-        modules => [Mod]
+    WorkerSup = #{
+        id => emqx_resource_worker_sup,
+        start => {emqx_resource_worker_sup, start_link, []},
+        restart => permanent,
+        shutdown => infinity,
+        type => supervisor
     },
-    case supervisor:start_child(emqx_resource_sup, Spec) of
-        {ok, _Pid} -> ok;
-        {error, {already_started, _}} -> ok;
-        {error, already_present} -> ok;
-        {error, _} = Err -> Err
-    end.
-
-ensure_worker_stopped(ResId, Idx) ->
-    ChildId = ?CHILD_ID(emqx_resource_worker, ResId, Idx),
-    case supervisor:terminate_child(emqx_resource_sup, ChildId) of
-        ok ->
-            supervisor:delete_child(emqx_resource_sup, ChildId);
-        {error, not_found} ->
-            ok;
-        {error, Reason} ->
-            {error, Reason}
-    end.
+    {ok, {SupFlags, [Metrics, ResourceManager, WorkerSup]}}.
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 7acf2d0f9..ae0d24313 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -44,6 +44,8 @@
 
 -export([running/3, blocked/3]).
 
+-export([queue_item_marshaller/1]).
+
 -define(RESUME_INTERVAL, 15000).
 
 %% count
@@ -51,11 +53,15 @@
 %% milliseconds
 -define(DEFAULT_BATCH_TIME, 10).
 
+-define(Q_ITEM(REQUEST), {q_item, REQUEST}).
+
 -define(QUERY(FROM, REQUEST), {FROM, REQUEST}).
 -define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}).
 -define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]).
 
--define(RESOURCE_ERROR(Reason, Msg), {error, {resource_error, #{reason => Reason, msg => Msg}}}).
+-define(RESOURCE_ERROR(Reason, Msg),
+    {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}}
+).
 -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
 
 -type id() :: binary().
@@ -72,21 +78,21 @@ callback_mode() -> [state_functions, state_enter].
 start_link(Id, Index, Opts) ->
     gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []).
 
--spec query(id(), request()) -> ok.
+-spec query(id(), request()) -> Result :: term().
 query(Id, Request) ->
-    gen_statem:call(pick(Id, self()), {query, Request}).
+    query(Id, self(), Request).
 
--spec query(id(), term(), request()) -> ok.
-query(Id, Key, Request) ->
-    gen_statem:call(pick(Id, Key), {query, Request}).
+-spec query(id(), term(), request()) -> Result :: term().
+query(Id, PickKey, Request) ->
+    pick_query(call, Id, PickKey, {query, Request}).
 
--spec query_async(id(), request(), reply_fun()) -> ok.
+-spec query_async(id(), request(), reply_fun()) -> Result :: term().
 query_async(Id, Request, ReplyFun) ->
-    gen_statem:cast(pick(Id, self()), {query, Request, ReplyFun}).
+    query_async(Id, self(), Request, ReplyFun).
 
--spec query_async(id(), term(), request(), reply_fun()) -> ok.
-query_async(Id, Key, Request, ReplyFun) ->
-    gen_statem:cast(pick(Id, Key), {query, Request, ReplyFun}).
+-spec query_async(id(), term(), request(), reply_fun()) -> Result :: term().
+query_async(Id, PickKey, Request, ReplyFun) ->
+    pick_query(cast, Id, PickKey, {query, Request, ReplyFun}).
 
 -spec block(pid() | atom()) -> ok.
 block(ServerRef) ->
@@ -97,17 +103,24 @@ resume(ServerRef) ->
     gen_statem:cast(ServerRef, resume).
 
 init({Id, Index, Opts}) ->
+    process_flag(trap_exit, true),
     true = gproc_pool:connect_worker(Id, {Id, Index}),
     BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE),
     Queue =
         case maps:get(queue_enabled, Opts, true) of
-            true -> replayq:open(#{dir => disk_queue_dir(Id, Index), seg_bytes => 10000000});
-            false -> undefined
+            true ->
+                replayq:open(#{
+                    dir => disk_queue_dir(Id, Index),
+                    seg_bytes => 10000000,
+                    marshaller => fun ?MODULE:queue_item_marshaller/1
+                });
+            false ->
+                undefined
         end,
     St = #{
         id => Id,
         index => Index,
-        batch_enabled => maps:get(batch_enabled, Opts, true),
+        batch_enabled => maps:get(batch_enabled, Opts, false),
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
         queue => Queue,
@@ -128,7 +141,7 @@ running(cast, {query, Request, ReplyFun}, St) ->
 running({call, From}, {query, Request}, St) ->
     query_or_acc(From, Request, St);
 running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) ->
-    {keep_state, flush(St#{tref := undefined})};
+    flush(St#{tref := undefined});
 running(info, {flush, _Ref}, _St) ->
     keep_state_and_data;
 running(info, Info, _St) ->
@@ -154,12 +167,21 @@ terminate(_Reason, #{id := Id, index := Index}) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
+queue_item_marshaller(?Q_ITEM(_) = I) ->
+    term_to_binary(I);
+queue_item_marshaller(Bin) when is_binary(Bin) ->
+    binary_to_term(Bin).
+
 %%==============================================================================
-pick(Id, Key) ->
-    Pid = gproc_pool:pick_worker(Id, Key),
-    case is_pid(Pid) of
-        true -> Pid;
-        false -> error({failed_to_pick_worker, {Id, Key}})
+pick_query(Fun, Id, Key, Query) ->
+    try gproc_pool:pick_worker(Id, Key) of
+        Pid when is_pid(Pid) ->
+            gen_statem:Fun(Pid, Query);
+        _ ->
+            ?RESOURCE_ERROR(not_created, "resource not found")
+    catch
+        error:badarg ->
+            ?RESOURCE_ERROR(not_created, "resource not found")
     end.
 
 do_resume(#{queue := undefined} = St) ->
@@ -168,9 +190,9 @@ do_resume(#{queue := Q, id := Id} = St) ->
     case replayq:peek(Q) of
         empty ->
             {next_state, running, St};
-        First ->
+        ?Q_ITEM(First) ->
             Result = call_query(Id, First),
-            case handle_query_result(Id, false, Result) of
+            case handle_query_result(Id, Result, false) of
                 %% Send failed because resource down
                 true ->
                     {keep_state, St, {state_timeout, ?RESUME_INTERVAL, resume}};
@@ -182,6 +204,11 @@ do_resume(#{queue := Q, id := Id} = St) ->
             end
     end.
 
+handle_blocked(From, Request, #{id := Id, queue := Q} = St) ->
+    Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
+    _ = reply_caller(Id, ?REPLY(From, Request, Error), false),
+    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}}.
+
 drop_head(Q) ->
     {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
     ok = replayq:ack(Q1, AckRef),
@@ -196,26 +223,21 @@ acc_query(From, Request, #{acc := Acc, acc_left := Left} = St0) ->
     Acc1 = [?QUERY(From, Request) | Acc],
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
-        true -> {keep_state, flush(St)};
+        true -> flush(St);
         false -> {keep_state, ensure_flush_timer(St)}
     end.
 
 send_query(From, Request, #{id := Id, queue := Q} = St) ->
     Result = call_query(Id, Request),
-    case reply_caller(Id, Q, ?REPLY(From, Request, Result)) of
+    case reply_caller(Id, ?REPLY(From, Request, Result), false) of
         true ->
-            {keep_state, St#{queue := maybe_append_queue(Q, [Request])}};
+            {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}};
         false ->
-            {next_state, blocked, St}
+            {keep_state, St}
     end.
 
-handle_blocked(From, Request, #{id := Id, queue := Q} = St) ->
-    Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
-    _ = reply_caller(Id, Q, ?REPLY(From, Request, Error)),
-    {keep_state, St#{queue := maybe_append_queue(Q, [Request])}}.
-
 flush(#{acc := []} = St) ->
-    St;
+    {keep_state, St};
 flush(
     #{
         id := Id,
@@ -228,65 +250,65 @@ flush(
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
     case batch_reply_caller(Id, BatchResults) of
         true ->
-            Q1 = maybe_append_queue(Q0, [Request || ?QUERY(_, Request) <- Batch]),
-            {keep_state, St1#{queue := Q1}};
+            Q1 = maybe_append_queue(Q0, [?Q_ITEM(Request) || ?QUERY(_, Request) <- Batch]),
+            {next_state, blocked, St1#{queue := Q1}};
         false ->
-            {next_state, blocked, St1}
+            {keep_state, St1}
     end.
 
-maybe_append_queue(undefined, _Query) -> undefined;
-maybe_append_queue(Q, Query) -> replayq:append(Q, Query).
+maybe_append_queue(undefined, _Request) -> undefined;
+maybe_append_queue(Q, Request) -> replayq:append(Q, Request).
 
 batch_reply_caller(Id, BatchResults) ->
     lists:foldl(
         fun(Reply, BlockWorker) ->
-            reply_caller(Id, BlockWorker, Reply)
+            reply_caller(Id, Reply, BlockWorker)
         end,
         false,
         BatchResults
     ).
 
-reply_caller(Id, BlockWorker, ?REPLY(undefined, _, Result)) ->
-    handle_query_result(Id, BlockWorker, Result);
-reply_caller(Id, BlockWorker, ?REPLY({ReplyFun, Args}, _, Result)) ->
+reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) ->
+    handle_query_result(Id, Result, BlockWorker);
+reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) ->
     ?SAFE_CALL(ReplyFun(Result, Args)),
-    handle_query_result(Id, BlockWorker, Result);
-reply_caller(Id, BlockWorker, ?REPLY(From, _, Result)) ->
+    handle_query_result(Id, Result, BlockWorker);
+reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
     gen_statem:reply(From, Result),
-    handle_query_result(Id, BlockWorker, Result).
+    handle_query_result(Id, Result, BlockWorker).
 
-handle_query_result(Id, BlockWorker, ok) ->
+handle_query_result(Id, ok, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, success),
     BlockWorker;
-handle_query_result(Id, BlockWorker, {ok, _}) ->
+handle_query_result(Id, {ok, _}, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, success),
     BlockWorker;
-handle_query_result(Id, BlockWorker, ?RESOURCE_ERROR_M(exception, _)) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(exception, _), BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, exception),
     BlockWorker;
-handle_query_result(_Id, _, ?RESOURCE_ERROR_M(NotWorking, _)) when
+handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _) when
     NotWorking == not_connected; NotWorking == blocked
 ->
     true;
-handle_query_result(_Id, BlockWorker, ?RESOURCE_ERROR_M(_, _)) ->
+handle_query_result(_Id, ?RESOURCE_ERROR_M(_, _), BlockWorker) ->
     BlockWorker;
-handle_query_result(Id, BlockWorker, {error, _}) ->
+handle_query_result(Id, {error, _}, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
     BlockWorker;
-handle_query_result(Id, _BlockWorker, {resource_down, _}) ->
+handle_query_result(Id, {resource_down, _}, _BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down),
     true.
 
 call_query(Id, Request) ->
-    do_call_query(on_query, Id, Request).
+    do_call_query(on_query, Id, Request, 1).
 
 call_batch_query(Id, Batch) ->
-    do_call_query(on_batch_query, Id, Batch).
+    do_call_query(on_batch_query, Id, Batch, length(Batch)).
 
-do_call_query(Fun, Id, Data) ->
+do_call_query(Fun, Id, Data, Count) ->
     case emqx_resource_manager:ets_lookup(Id) of
         {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
-            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Data)),
+            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, Count),
             try Mod:Fun(Id, Data, ResourceState) of
                 %% if the callback module (connector) wants to return an error that
                 %% makes the current resource goes into the `error` state, it should
diff --git a/apps/emqx_resource/src/emqx_resource_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_worker_sup.erl
new file mode 100644
index 000000000..a2b3a1ba5
--- /dev/null
+++ b/apps/emqx_resource/src/emqx_resource_worker_sup.erl
@@ -0,0 +1,136 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_resource_worker_sup).
+-behaviour(supervisor).
+
+%%%=============================================================================
+%%% Exports and Definitions
+%%%=============================================================================
+
+%% External API
+-export([start_link/0]).
+
+-export([start_workers/2, stop_workers/2]).
+
+%% Callbacks
+-export([init/1]).
+
+-define(SERVER, ?MODULE).
+
+%%%=============================================================================
+%%% API
+%%%=============================================================================
+
+-spec start_link() -> supervisor:startlink_ret().
+start_link() ->
+    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+%%%=============================================================================
+%%% Callbacks
+%%%=============================================================================
+
+-spec init(list()) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}} | ignore.
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_one,
+        intensity => 100,
+        period => 30
+    },
+    ChildSpecs = [],
+    {ok, {SupFlags, ChildSpecs}}.
+
+start_workers(ResId, Opts) ->
+    PoolSize = pool_size(Opts),
+    _ = ensure_worker_pool(ResId, hash, [{size, PoolSize}]),
+    lists:foreach(
+        fun(Idx) ->
+            _ = ensure_worker_added(ResId, Idx),
+            ok = ensure_worker_started(ResId, Idx, Opts)
+        end,
+        lists:seq(1, PoolSize)
+    ).
+
+stop_workers(ResId, Opts) ->
+    PoolSize = pool_size(Opts),
+    lists:foreach(
+        fun(Idx) ->
+            ensure_worker_removed(ResId, Idx)
+        end,
+        lists:seq(1, PoolSize)
+    ),
+    ensure_worker_pool_removed(ResId),
+    ok.
+
+%%%=============================================================================
+%%% Internal
+%%%=============================================================================
+pool_size(Opts) ->
+    maps:get(worker_pool_size, Opts, erlang:system_info(schedulers_online)).
+
+ensure_worker_pool(ResId, Type, Opts) ->
+    try
+        gproc_pool:new(ResId, Type, Opts)
+    catch
+        error:exists -> ok
+    end,
+    ok.
+
+ensure_worker_added(ResId, Idx) ->
+    try
+        gproc_pool:add_worker(ResId, {ResId, Idx}, Idx)
+    catch
+        error:exists -> ok
+    end,
+    ok.
+
+-define(CHILD_ID(MOD, RESID, INDEX), {MOD, RESID, INDEX}).
+ensure_worker_started(ResId, Idx, Opts) ->
+    Mod = emqx_resource_worker,
+    Spec = #{
+        id => ?CHILD_ID(Mod, ResId, Idx),
+        start => {Mod, start_link, [ResId, Idx, Opts]},
+        restart => transient,
+        shutdown => 5000,
+        type => worker,
+        modules => [Mod]
+    },
+    case supervisor:start_child(emqx_resource_sup, Spec) of
+        {ok, _Pid} -> ok;
+        {error, {already_started, _}} -> ok;
+        {error, already_present} -> ok;
+        {error, _} = Err -> Err
+    end.
+
+ensure_worker_removed(ResId, Idx) ->
+    ChildId = ?CHILD_ID(emqx_resource_worker, ResId, Idx),
+    case supervisor:terminate_child(emqx_resource_sup, ChildId) of
+        ok ->
+            Res = supervisor:delete_child(emqx_resource_sup, ChildId),
+            _ = gproc_pool:remove_worker(ResId, {ResId, Idx}),
+            Res;
+        {error, not_found} ->
+            ok;
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+ensure_worker_pool_removed(ResId) ->
+    try
+        gproc_pool:delete(ResId)
+    catch
+        error:badarg -> ok
+    end,
+    ok.
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 51e6bac43..915c59611 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -26,6 +26,7 @@
 -define(TEST_RESOURCE, emqx_test_resource).
 -define(ID, <<"id">>).
 -define(DEFAULT_RESOURCE_GROUP, <<"default">>).
+-define(RESOURCE_ERROR(REASON), {error, {resource_error, #{reason := REASON}}}).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -80,7 +81,7 @@ t_create_remove(_) ->
         #{name => test_resource},
         #{}
     ),
-    #{pid := Pid} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state),
 
     ?assert(is_process_alive(Pid)),
 
@@ -110,7 +111,7 @@ t_create_remove_local(_) ->
         #{name => test_resource},
         #{}
     ),
-    #{pid := Pid} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state),
 
     ?assert(is_process_alive(Pid)),
 
@@ -127,7 +128,7 @@ t_create_remove_local(_) ->
     {error, _} = emqx_resource:remove_local(?ID),
 
     ?assertMatch(
-        {error, {emqx_resource, #{reason := not_found}}},
+        ?RESOURCE_ERROR(not_created),
         emqx_resource:query(?ID, get_state)
     ),
     ?assertNot(is_process_alive(Pid)).
@@ -143,23 +144,23 @@ t_do_not_start_after_created(_) ->
     %% the resource should remain `disconnected` after created
     timer:sleep(200),
     ?assertMatch(
-        {error, {emqx_resource, #{reason := not_connected}}},
+        ?RESOURCE_ERROR(stopped),
         emqx_resource:query(?ID, get_state)
     ),
     ?assertMatch(
-        {ok, _, #{status := disconnected}},
+        {ok, _, #{status := stopped}},
         emqx_resource:get_instance(?ID)
     ),
 
     %% start the resource manually..
     ok = emqx_resource:start(?ID),
-    #{pid := Pid} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state),
     ?assert(is_process_alive(Pid)),
 
     %% restart the resource
     ok = emqx_resource:restart(?ID),
     ?assertNot(is_process_alive(Pid)),
-    #{pid := Pid2} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid2}} = emqx_resource:query(?ID, get_state),
     ?assert(is_process_alive(Pid2)),
 
     ok = emqx_resource:remove_local(?ID),
@@ -174,23 +175,10 @@ t_query(_) ->
         #{name => test_resource}
     ),
 
-    Pid = self(),
-    Success = fun() -> Pid ! success end,
-    Failure = fun() -> Pid ! failure end,
-
-    #{pid := _} = emqx_resource:query(?ID, get_state),
-    #{pid := _} = emqx_resource:query(?ID, get_state, {[{Success, []}], [{Failure, []}]}),
-    #{pid := _} = emqx_resource:query(?ID, get_state, undefined),
-    #{pid := _} = emqx_resource:query(?ID, get_state_failed, undefined),
-
-    receive
-        Message -> ?assertEqual(success, Message)
-    after 100 ->
-        ?assert(false)
-    end,
+    {ok, #{pid := _}} = emqx_resource:query(?ID, get_state),
 
     ?assertMatch(
-        {error, {emqx_resource, #{reason := not_found}}},
+        ?RESOURCE_ERROR(not_created),
         emqx_resource:query(<<"unknown">>, get_state)
     ),
 
@@ -201,11 +189,14 @@ t_healthy_timeout(_) ->
         ?ID,
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
-        #{name => <<"test_resource">>},
-        #{health_check_timeout => 200}
+        #{name => <<"bad_not_atom_name">>, register => true},
+        %% the ?TEST_RESOURCE always returns the `Mod:on_get_status/2` 300ms later.
+        #{health_check_interval => 200}
+    ),
+    ?assertMatch(
+        ?RESOURCE_ERROR(not_connected),
+        emqx_resource:query(?ID, get_state)
     ),
-    timer:sleep(500),
-
     ok = emqx_resource:remove_local(?ID).
 
 t_healthy(_) ->
@@ -213,11 +204,9 @@ t_healthy(_) ->
         ?ID,
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
-        #{name => <<"test_resource">>}
+        #{name => test_resource}
     ),
-    timer:sleep(400),
-
-    #{pid := Pid} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state),
     timer:sleep(300),
     emqx_resource:set_resource_status_connecting(?ID),
 
@@ -229,10 +218,10 @@ t_healthy(_) ->
 
     erlang:exit(Pid, shutdown),
 
-    ?assertEqual({ok, connecting}, emqx_resource:health_check(?ID)),
+    ?assertEqual({ok, disconnected}, emqx_resource:health_check(?ID)),
 
     ?assertMatch(
-        [#{status := connecting}],
+        [#{status := disconnected}],
         emqx_resource:list_instances_verbose()
     ),
 
@@ -260,7 +249,7 @@ t_stop_start(_) ->
         #{}
     ),
 
-    #{pid := Pid0} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state),
 
     ?assert(is_process_alive(Pid0)),
 
@@ -269,14 +258,14 @@ t_stop_start(_) ->
     ?assertNot(is_process_alive(Pid0)),
 
     ?assertMatch(
-        {error, {emqx_resource, #{reason := not_connected}}},
+        ?RESOURCE_ERROR(stopped),
         emqx_resource:query(?ID, get_state)
     ),
 
     ok = emqx_resource:restart(?ID),
     timer:sleep(300),
 
-    #{pid := Pid1} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state),
 
     ?assert(is_process_alive(Pid1)).
 
@@ -302,7 +291,7 @@ t_stop_start_local(_) ->
         #{}
     ),
 
-    #{pid := Pid0} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state),
 
     ?assert(is_process_alive(Pid0)),
 
@@ -311,13 +300,13 @@ t_stop_start_local(_) ->
     ?assertNot(is_process_alive(Pid0)),
 
     ?assertMatch(
-        {error, {emqx_resource, #{reason := not_connected}}},
+        ?RESOURCE_ERROR(stopped),
         emqx_resource:query(?ID, get_state)
     ),
 
     ok = emqx_resource:restart(?ID),
 
-    #{pid := Pid1} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state),
 
     ?assert(is_process_alive(Pid1)).
 
@@ -368,17 +357,17 @@ create_dry_run_local_succ() ->
     ?assertEqual(undefined, whereis(test_resource)).
 
 t_create_dry_run_local_failed(_) ->
-    {Res1, _} = emqx_resource:create_dry_run_local(
+    Res1 = emqx_resource:create_dry_run_local(
         ?TEST_RESOURCE,
-        #{cteate_error => true}
+        #{create_error => true}
     ),
-    ?assertEqual(error, Res1),
+    ?assertMatch({error, _}, Res1),
 
-    {Res2, _} = emqx_resource:create_dry_run_local(
+    Res2 = emqx_resource:create_dry_run_local(
         ?TEST_RESOURCE,
         #{name => test_resource, health_check_error => true}
     ),
-    ?assertEqual(error, Res2),
+    ?assertMatch({error, _}, Res2),
 
     Res3 = emqx_resource:create_dry_run_local(
         ?TEST_RESOURCE,
@@ -400,7 +389,7 @@ t_reset_metrics(_) ->
         #{name => test_resource}
     ),
 
-    #{pid := Pid} = emqx_resource:query(?ID, get_state),
+    {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state),
     emqx_resource:reset_metrics(?ID),
     ?assert(is_process_alive(Pid)),
     ok = emqx_resource:remove(?ID),
diff --git a/apps/emqx_resource/test/emqx_test_resource.erl b/apps/emqx_resource/test/emqx_test_resource.erl
index c23f87d50..569579d27 100644
--- a/apps/emqx_resource/test/emqx_test_resource.erl
+++ b/apps/emqx_resource/test/emqx_test_resource.erl
@@ -24,7 +24,7 @@
 -export([
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -50,24 +50,20 @@ on_start(_InstId, #{create_error := true}) ->
     error("some error");
 on_start(InstId, #{name := Name, stop_error := true} = Opts) ->
     Register = maps:get(register, Opts, false),
-    {ok, #{
-        name => Name,
+    {ok, Opts#{
         id => InstId,
         stop_error => true,
         pid => spawn_dummy_process(Name, Register)
     }};
-on_start(InstId, #{name := Name, health_check_error := true} = Opts) ->
+on_start(InstId, #{name := Name} = Opts) ->
     Register = maps:get(register, Opts, false),
-    {ok, #{
-        name => Name,
+    {ok, Opts#{
         id => InstId,
-        health_check_error => true,
         pid => spawn_dummy_process(Name, Register)
     }};
 on_start(InstId, #{name := Name} = Opts) ->
     Register = maps:get(register, Opts, false),
-    {ok, #{
-        name => Name,
+    {ok, Opts#{
         id => InstId,
         pid => spawn_dummy_process(Name, Register)
     }}.
@@ -78,12 +74,10 @@ on_stop(_InstId, #{pid := Pid}) ->
     erlang:exit(Pid, shutdown),
     ok.
 
-on_query(_InstId, get_state, AfterQuery, State) ->
-    emqx_resource:query_success(AfterQuery),
-    State;
-on_query(_InstId, get_state_failed, AfterQuery, State) ->
-    emqx_resource:query_failed(AfterQuery),
-    State.
+on_query(_InstId, get_state, State) ->
+    {ok, State};
+on_query(_InstId, get_state_failed, State) ->
+    {error, State}.
 
 on_get_status(_InstId, #{health_check_error := true}) ->
     disconnected;
@@ -91,10 +85,11 @@ on_get_status(_InstId, #{pid := Pid}) ->
     timer:sleep(300),
     case is_process_alive(Pid) of
         true -> connected;
-        false -> connecting
+        false -> disconnected
     end.
 
 spawn_dummy_process(Name, Register) ->
+    ct:pal("---- Register Name: ~p", [Name]),
     spawn(
         fun() ->
             true =

From 0377d3cf61469aa51cbe367bdfb5dd1a2768e518 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 29 Jul 2022 18:49:57 +0800
Subject: [PATCH 032/232] fix: update existing testcases for new emqx_resource

---
 .../src/simple_authn/emqx_authn_mongodb.erl   |   2 +-
 apps/emqx_authz/src/emqx_authz_mongodb.erl    |   4 +-
 .../test/emqx_bridge_api_SUITE.erl            | 116 ++++++++++--------
 .../src/emqx_connector_mqtt.erl               |   5 +-
 .../test/emqx_connector_mongo_SUITE.erl       |   8 +-
 apps/emqx_resource/src/emqx_resource.app.src  |   2 +-
 apps/emqx_resource/src/emqx_resource.erl      |   5 +-
 .../src/emqx_resource_worker.erl              |  26 ++--
 8 files changed, 98 insertions(+), 70 deletions(-)

diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
index ff9c97717..1351ae0dd 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
@@ -164,7 +164,7 @@ authenticate(
 ) ->
     Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential),
     case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of
-        undefined ->
+        {ok, undefined} ->
             ignore;
         {error, Reason} ->
             ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{
diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl
index ac450e4cc..a1e1b8136 100644
--- a/apps/emqx_authz/src/emqx_authz_mongodb.erl
+++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl
@@ -92,9 +92,9 @@ authorize(
                 resource_id => ResourceID
             }),
             nomatch;
-        [] ->
+        {ok, []} ->
             nomatch;
-        Rows ->
+        {ok, Rows} ->
             Rules = [
                 emqx_authz_rule:compile({Permission, all, Action, Topics})
              || #{
diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
index c048a13fe..9346fb9c0 100644
--- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
@@ -24,7 +24,7 @@
 -include_lib("common_test/include/ct.hrl").
 -define(CONF_DEFAULT, <<"bridges: {}">>).
 -define(BRIDGE_TYPE, <<"webhook">>).
--define(BRIDGE_NAME, <<"test_bridge">>).
+-define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))).
 -define(URL(PORT, PATH),
     list_to_binary(
         io_lib:format(
@@ -78,8 +78,12 @@ set_special_configs(_) ->
 
 init_per_testcase(_, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
-    Config.
-end_per_testcase(_, _Config) ->
+    {Port, Sock, Acceptor} = start_http_server(fun handle_fun_200_ok/2),
+    [{port, Port}, {sock, Sock}, {acceptor, Acceptor} | Config].
+end_per_testcase(_, Config) ->
+    Sock = ?config(sock, Config),
+    Acceptor = ?config(acceptor, Config),
+    stop_http_server(Sock, Acceptor),
     clear_resources(),
     ok.
 
@@ -95,31 +99,39 @@ clear_resources() ->
 %% HTTP server for testing
 %%------------------------------------------------------------------------------
 start_http_server(HandleFun) ->
+    process_flag(trap_exit, true),
     Parent = self(),
-    spawn_link(fun() ->
-        {Port, Sock} = listen_on_random_port(),
-        Parent ! {port, Port},
-        loop(Sock, HandleFun, Parent)
+    {Port, Sock} = listen_on_random_port(),
+    Acceptor = spawn_link(fun() ->
+        accept_loop(Sock, HandleFun, Parent)
     end),
-    receive
-        {port, Port} -> Port
-    after 2000 -> error({timeout, start_http_server})
-    end.
+    timer:sleep(100),
+    {Port, Sock, Acceptor}.
+
+stop_http_server(Sock, Acceptor) ->
+    exit(Acceptor, kill),
+    gen_tcp:close(Sock).
 
 listen_on_random_port() ->
     Min = 1024,
     Max = 65000,
+    rand:seed(exsplus, erlang:timestamp()),
     Port = rand:uniform(Max - Min) + Min,
-    case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of
+    case
+        gen_tcp:listen(Port, [
+            binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}
+        ])
+    of
         {ok, Sock} -> {Port, Sock};
         {error, eaddrinuse} -> listen_on_random_port()
     end.
 
-loop(Sock, HandleFun, Parent) ->
+accept_loop(Sock, HandleFun, Parent) ->
+    process_flag(trap_exit, true),
     {ok, Conn} = gen_tcp:accept(Sock),
-    Handler = spawn(fun() -> HandleFun(Conn, Parent) end),
+    Handler = spawn_link(fun() -> HandleFun(Conn, Parent) end),
     gen_tcp:controlling_process(Conn, Handler),
-    loop(Sock, HandleFun, Parent).
+    accept_loop(Sock, HandleFun, Parent).
 
 make_response(CodeStr, Str) ->
     B = iolist_to_binary(Str),
@@ -138,7 +150,9 @@ handle_fun_200_ok(Conn, Parent) ->
             Parent ! {http_server, received, Req},
             gen_tcp:send(Conn, make_response("200 OK", "Request OK")),
             handle_fun_200_ok(Conn, Parent);
-        {error, closed} ->
+        {error, Reason} ->
+            ct:pal("the http handler recv error: ~p", [Reason]),
+            timer:sleep(100),
             gen_tcp:close(Conn)
     end.
 
@@ -153,24 +167,25 @@ parse_http_request(ReqStr0) ->
 %% Testcases
 %%------------------------------------------------------------------------------
 
-t_http_crud_apis(_) ->
-    Port = start_http_server(fun handle_fun_200_ok/2),
+t_http_crud_apis(Config) ->
+    Port = ?config(port, Config),
     %% assert we there's no bridges at first
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
     %% then we add a webhook bridge, using POST
     %% POST /bridges/ will create a bridge
     URL1 = ?URL(Port, "path1"),
+    Name = ?BRIDGE_NAME,
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
     ),
 
     %ct:pal("---bridge: ~p", [Bridge]),
     #{
         <<"type">> := ?BRIDGE_TYPE,
-        <<"name">> := ?BRIDGE_NAME,
+        <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := _,
         <<"node_status">> := [_ | _],
@@ -179,7 +194,7 @@ t_http_crud_apis(_) ->
         <<"url">> := URL1
     } = jsx:decode(Bridge),
 
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
     %% send an message to emqx and the message should be forwarded to the HTTP server
     Body = <<"my msg">>,
     emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)),
@@ -203,12 +218,12 @@ t_http_crud_apis(_) ->
     {ok, 200, Bridge2} = request(
         put,
         uri(["bridges", BridgeID]),
-        ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name)
     ),
     ?assertMatch(
         #{
             <<"type">> := ?BRIDGE_TYPE,
-            <<"name">> := ?BRIDGE_NAME,
+            <<"name">> := Name,
             <<"enable">> := true,
             <<"status">> := _,
             <<"node_status">> := [_ | _],
@@ -225,7 +240,7 @@ t_http_crud_apis(_) ->
         [
             #{
                 <<"type">> := ?BRIDGE_TYPE,
-                <<"name">> := ?BRIDGE_NAME,
+                <<"name">> := Name,
                 <<"enable">> := true,
                 <<"status">> := _,
                 <<"node_status">> := [_ | _],
@@ -242,7 +257,7 @@ t_http_crud_apis(_) ->
     ?assertMatch(
         #{
             <<"type">> := ?BRIDGE_TYPE,
-            <<"name">> := ?BRIDGE_NAME,
+            <<"name">> := Name,
             <<"enable">> := true,
             <<"status">> := _,
             <<"node_status">> := [_ | _],
@@ -275,7 +290,7 @@ t_http_crud_apis(_) ->
     {ok, 404, ErrMsg2} = request(
         put,
         uri(["bridges", BridgeID]),
-        ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name)
     ),
     ?assertMatch(
         #{
@@ -286,29 +301,28 @@ t_http_crud_apis(_) ->
     ),
     ok.
 
-t_start_stop_bridges(_) ->
-    lists:foreach(
-        fun(Type) ->
-            do_start_stop_bridges(Type)
-        end,
-        [node, cluster]
-    ).
+t_start_stop_bridges_node(Config) ->
+    do_start_stop_bridges(node, Config).
 
-do_start_stop_bridges(Type) ->
+t_start_stop_bridges_cluster(Config) ->
+    do_start_stop_bridges(cluster, Config).
+
+do_start_stop_bridges(Type, Config) ->
     %% assert we there's no bridges at first
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
-    Port = start_http_server(fun handle_fun_200_ok/2),
+    Port = ?config(port, Config),
     URL1 = ?URL(Port, "abc"),
+    Name = atom_to_binary(Type),
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
     ),
     %ct:pal("the bridge ==== ~p", [Bridge]),
     #{
         <<"type">> := ?BRIDGE_TYPE,
-        <<"name">> := ?BRIDGE_NAME,
+        <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := <<"connected">>,
         <<"node_status">> := [_ | _],
@@ -316,11 +330,11 @@ do_start_stop_bridges(Type) ->
         <<"node_metrics">> := [_ | _],
         <<"url">> := URL1
     } = jsx:decode(Bridge),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
     %% stop it
     {ok, 200, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>),
     {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
-    ?assertMatch(#{<<"status">> := <<"disconnected">>}, jsx:decode(Bridge2)),
+    ?assertMatch(#{<<"status">> := <<"stopped">>}, jsx:decode(Bridge2)),
     %% start again
     {ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>),
     {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
@@ -339,21 +353,22 @@ do_start_stop_bridges(Type) ->
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
 
-t_enable_disable_bridges(_) ->
+t_enable_disable_bridges(Config) ->
     %% assert we there's no bridges at first
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
-    Port = start_http_server(fun handle_fun_200_ok/2),
+    Name = ?BRIDGE_NAME,
+    Port = ?config(port, Config),
     URL1 = ?URL(Port, "abc"),
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
     ),
     %ct:pal("the bridge ==== ~p", [Bridge]),
     #{
         <<"type">> := ?BRIDGE_TYPE,
-        <<"name">> := ?BRIDGE_NAME,
+        <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := <<"connected">>,
         <<"node_status">> := [_ | _],
@@ -361,11 +376,11 @@ t_enable_disable_bridges(_) ->
         <<"node_metrics">> := [_ | _],
         <<"url">> := URL1
     } = jsx:decode(Bridge),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
     %% disable it
     {ok, 200, <<>>} = request(post, operation_path(cluster, disable, BridgeID), <<"">>),
     {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
-    ?assertMatch(#{<<"status">> := <<"disconnected">>}, jsx:decode(Bridge2)),
+    ?assertMatch(#{<<"status">> := <<"stopped">>}, jsx:decode(Bridge2)),
     %% enable again
     {ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>),
     {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
@@ -391,21 +406,22 @@ t_enable_disable_bridges(_) ->
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
 
-t_reset_bridges(_) ->
+t_reset_bridges(Config) ->
     %% assert we there's no bridges at first
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
-    Port = start_http_server(fun handle_fun_200_ok/2),
+    Name = ?BRIDGE_NAME,
+    Port = ?config(port, Config),
     URL1 = ?URL(Port, "abc"),
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
     ),
     %ct:pal("the bridge ==== ~p", [Bridge]),
     #{
         <<"type">> := ?BRIDGE_TYPE,
-        <<"name">> := ?BRIDGE_NAME,
+        <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := <<"connected">>,
         <<"node_status">> := [_ | _],
@@ -413,7 +429,7 @@ t_reset_bridges(_) ->
         <<"node_metrics">> := [_ | _],
         <<"url">> := URL1
     } = jsx:decode(Bridge),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
     {ok, 200, <<"Reset success">>} = request(put, uri(["bridges", BridgeID, "reset_metrics"]), []),
 
     %% delete the bridge
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 98635de3f..0957e3c18 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -134,7 +134,8 @@ drop_bridge(Name) ->
 %% When use this bridge as a data source, ?MODULE:on_message_received will be called
 %% if the bridge received msgs from the remote broker.
 on_message_received(Msg, HookPoint, InstId) ->
-    _ = emqx_resource:query(InstId, {message_received, Msg}),
+    emqx_resource:inc_matched(InstId),
+    emqx_resource:inc_success(InstId),
     emqx:run_hook(HookPoint, [Msg]).
 
 %% ===================================================================
@@ -181,8 +182,6 @@ on_stop(_InstId, #{name := InstanceId}) ->
             })
     end.
 
-on_query(_InstId, {message_received, _Msg}, _State) ->
-    ok;
 on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) ->
     ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
     emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
index d76b8420a..2ad1f5f8e 100644
--- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
@@ -85,8 +85,8 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
         emqx_resource:get_instance(PoolName),
     ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)),
     % % Perform query as further check that the resource is working as expected
-    ?assertMatch([], emqx_resource:query(PoolName, test_query_find())),
-    ?assertMatch(undefined, emqx_resource:query(PoolName, test_query_find_one())),
+    ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())),
+    ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())),
     ?assertEqual(ok, emqx_resource:stop(PoolName)),
     % Resource will be listed still, but state will be changed and healthcheck will fail
     % as the worker no longer exists.
@@ -108,8 +108,8 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
     {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} =
         emqx_resource:get_instance(PoolName),
     ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)),
-    ?assertMatch([], emqx_resource:query(PoolName, test_query_find())),
-    ?assertMatch(undefined, emqx_resource:query(PoolName, test_query_find_one())),
+    ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())),
+    ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())),
     % Stop and remove the resource in one go.
     ?assertEqual(ok, emqx_resource:remove_local(PoolName)),
     ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src
index 1bfd02323..b688e3c11 100644
--- a/apps/emqx_resource/src/emqx_resource.app.src
+++ b/apps/emqx_resource/src/emqx_resource.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_resource, [
     {description, "Manager for all external resources"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_resource_app, []}},
     {applications, [
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 081264315..af047060c 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -103,7 +103,7 @@
     list_group_instances/1
 ]).
 
--export([inc_metrics_funcs/1, inc_success/1, inc_failed/1]).
+-export([inc_metrics_funcs/1, inc_matched/1, inc_success/1, inc_failed/1]).
 
 -optional_callbacks([
     on_query/3,
@@ -393,6 +393,9 @@ check_and_do(ResourceType, RawConfig, Do) when is_function(Do) ->
 
 %% =================================================================================
 
+inc_matched(ResId) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ResId, matched).
+
 inc_success(ResId) ->
     emqx_metrics_worker:inc(?RES_METRICS, ResId, success).
 
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index ae0d24313..9ab7fb749 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -44,7 +44,7 @@
 
 -export([running/3, blocked/3]).
 
--export([queue_item_marshaller/1]).
+-export([queue_item_marshaller/1, estimate_size/1]).
 
 -define(RESUME_INTERVAL, 15000).
 
@@ -112,6 +112,7 @@ init({Id, Index, Opts}) ->
                 replayq:open(#{
                     dir => disk_queue_dir(Id, Index),
                     seg_bytes => 10000000,
+                    sizer => fun ?MODULE:estimate_size/1,
                     marshaller => fun ?MODULE:queue_item_marshaller/1
                 });
             false ->
@@ -172,6 +173,9 @@ queue_item_marshaller(?Q_ITEM(_) = I) ->
 queue_item_marshaller(Bin) when is_binary(Bin) ->
     binary_to_term(Bin).
 
+estimate_size(QItem) ->
+    size(queue_item_marshaller(QItem)).
+
 %%==============================================================================
 pick_query(Fun, Id, Key, Query) ->
     try gproc_pool:pick_worker(Id, Key) of
@@ -277,12 +281,6 @@ reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
     gen_statem:reply(From, Result),
     handle_query_result(Id, Result, BlockWorker).
 
-handle_query_result(Id, ok, BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, success),
-    BlockWorker;
-handle_query_result(Id, {ok, _}, BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, success),
-    BlockWorker;
 handle_query_result(Id, ?RESOURCE_ERROR_M(exception, _), BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, exception),
     BlockWorker;
@@ -297,7 +295,12 @@ handle_query_result(Id, {error, _}, BlockWorker) ->
     BlockWorker;
 handle_query_result(Id, {resource_down, _}, _BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down),
-    true.
+    true;
+handle_query_result(Id, Result, BlockWorker) ->
+    %% assert
+    true = is_ok_result(Result),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, success),
+    BlockWorker.
 
 call_query(Id, Request) ->
     do_call_query(on_query, Id, Request, 1).
@@ -339,6 +342,13 @@ maybe_expand_batch_result(Result, Batch) ->
 
 %%==============================================================================
 
+is_ok_result(ok) ->
+    true;
+is_ok_result(R) when is_tuple(R) ->
+    erlang:element(1, R) == ok;
+is_ok_result(_) ->
+    false.
+
 -spec name(id(), integer()) -> atom().
 name(Id, Index) ->
     Mod = atom_to_list(?MODULE),

From d3950b9534ae9e220ad7e6a550f3f549ca4baf19 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 29 Jul 2022 23:26:54 +0800
Subject: [PATCH 033/232] fix(resource): make option 'queue_enabled' disabled
 by default

---
 apps/emqx_authn/src/emqx_authn.app.src        |  2 +-
 apps/emqx_authz/src/emqx_authz.app.src        |  2 +-
 .../emqx_connector/src/emqx_connector.app.src |  2 +-
 .../src/emqx_resource_worker.erl              | 27 +++++++------------
 .../test/emqx_resource_SUITE.erl              |  4 +++
 5 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index 8087e822f..ef67b9a14 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]},
diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index ed19b15a8..e40b5e64c 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src
index 007962da3..cce266966 100644
--- a/apps/emqx_connector/src/emqx_connector.app.src
+++ b/apps/emqx_connector/src/emqx_connector.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "An OTP application"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 9ab7fb749..055fbfc53 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -73,7 +73,7 @@
 -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) ->
     {{from(), result()}, NewCbState :: term()}.
 
-callback_mode() -> [state_functions, state_enter].
+callback_mode() -> [state_functions].
 
 start_link(Id, Index, Opts) ->
     gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []).
@@ -107,7 +107,7 @@ init({Id, Index, Opts}) ->
     true = gproc_pool:connect_worker(Id, {Id, Index}),
     BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE),
     Queue =
-        case maps:get(queue_enabled, Opts, true) of
+        case maps:get(queue_enabled, Opts, false) of
             true ->
                 replayq:open(#{
                     dir => disk_queue_dir(Id, Index),
@@ -131,8 +131,6 @@ init({Id, Index, Opts}) ->
     },
     {ok, blocked, St, {next_event, cast, resume}}.
 
-running(enter, _, _St) ->
-    keep_state_and_data;
 running(cast, resume, _St) ->
     keep_state_and_data;
 running(cast, block, St) ->
@@ -149,8 +147,6 @@ running(info, Info, _St) ->
     ?SLOG(error, #{msg => unexpected_msg, info => Info}),
     keep_state_and_data.
 
-blocked(enter, _, _St) ->
-    keep_state_and_data;
 blocked(cast, block, _St) ->
     keep_state_and_data;
 blocked(cast, resume, St) ->
@@ -218,28 +214,25 @@ drop_head(Q) ->
     ok = replayq:ack(Q1, AckRef),
     Q1.
 
-query_or_acc(From, Request, #{batch_enabled := true} = St) ->
-    acc_query(From, Request, St);
-query_or_acc(From, Request, #{batch_enabled := false} = St) ->
-    send_query(From, Request, St).
-
-acc_query(From, Request, #{acc := Acc, acc_left := Left} = St0) ->
+query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Left} = St0) ->
     Acc1 = [?QUERY(From, Request) | Acc],
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
         true -> flush(St);
         false -> {keep_state, ensure_flush_timer(St)}
-    end.
-
-send_query(From, Request, #{id := Id, queue := Q} = St) ->
-    Result = call_query(Id, Request),
-    case reply_caller(Id, ?REPLY(From, Request, Result), false) of
+    end;
+query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id} = St) ->
+    case send_query(From, Request, Id) of
         true ->
             {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}};
         false ->
             {keep_state, St}
     end.
 
+send_query(From, Request, Id) ->
+    Result = call_query(Id, Request),
+    reply_caller(Id, ?REPLY(From, Request, Result), false).
+
 flush(#{acc := []} = St) ->
     {keep_state, St};
 flush(
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 915c59611..278f556ef 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -347,6 +347,10 @@ t_create_dry_run_local(_) ->
     [] = ets:match(emqx_resource_manager, {{owner, '$1'}, '_'}).
 
 create_dry_run_local_succ() ->
+    case whereis(test_resource) of
+        undefined -> ok;
+        Pid -> exit(Pid, kill)
+    end,
     ?assertEqual(
         ok,
         emqx_resource:create_dry_run_local(

From 75adba07813de06bbf50196943bc5919e6f296c9 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Sat, 30 Jul 2022 16:35:12 +0800
Subject: [PATCH 034/232] fix: increase resource metrics using the resource id

---
 .../emqx_connector/src/emqx_connector_mqtt.erl |  9 +++++----
 .../src/emqx_resource_manager.erl              | 18 +++++++++++++-----
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 0957e3c18..66682caeb 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -133,9 +133,9 @@ drop_bridge(Name) ->
 %% ===================================================================
 %% When use this bridge as a data source, ?MODULE:on_message_received will be called
 %% if the bridge received msgs from the remote broker.
-on_message_received(Msg, HookPoint, InstId) ->
-    emqx_resource:inc_matched(InstId),
-    emqx_resource:inc_success(InstId),
+on_message_received(Msg, HookPoint, ResId) ->
+    emqx_resource:inc_matched(ResId),
+    emqx_resource:inc_success(ResId),
     emqx:run_hook(HookPoint, [Msg]).
 
 %% ===================================================================
@@ -206,11 +206,12 @@ make_sub_confs(EmptyMap, _) when map_size(EmptyMap) == 0 ->
 make_sub_confs(undefined, _) ->
     undefined;
 make_sub_confs(SubRemoteConf, InstId) ->
+    ResId = emqx_resource_manager:manager_id_to_resource_id(InstId),
     case maps:take(hookpoint, SubRemoteConf) of
         error ->
             SubRemoteConf;
         {HookPoint, SubConf} ->
-            MFA = {?MODULE, on_message_received, [HookPoint, InstId]},
+            MFA = {?MODULE, on_message_received, [HookPoint, ResId]},
             SubConf#{on_message_received => MFA}
     end.
 
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index b5bcbd330..7dad85d5b 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -38,8 +38,12 @@
     list_group/1,
     ets_lookup/1,
     get_metrics/1,
-    reset_metrics/1,
-    set_resource_status_connecting/1
+    reset_metrics/1
+]).
+
+-export([
+    set_resource_status_connecting/1,
+    manager_id_to_resource_id/1
 ]).
 
 % Server
@@ -64,6 +68,13 @@
 %% API
 %%------------------------------------------------------------------------------
 
+make_manager_id(ResId) ->
+    emqx_resource:generate_id(ResId).
+
+manager_id_to_resource_id(MgrId) ->
+    [ResId, _Index] = string:split(MgrId, ":", trailing),
+    ResId.
+
 %% @doc Called from emqx_resource when starting a resource instance.
 %%
 %% Triggers the emqx_resource_manager_sup supervisor to actually create
@@ -455,9 +466,6 @@ stop_resource(Data) ->
     _ = maybe_clear_alarm(Data#data.id),
     ok.
 
-make_manager_id(ResId) ->
-    emqx_resource:generate_id(ResId).
-
 make_test_id() ->
     RandId = iolist_to_binary(emqx_misc:gen_id(16)),
     <<?TEST_ID_PREFIX, RandId/binary>>.

From 8f0954837bdca5037bac40f799c531c8fe3e7a9e Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Sat, 30 Jul 2022 22:42:26 +0800
Subject: [PATCH 035/232] fix: update test cases for emqx_connector

---
 apps/emqx/src/emqx.app.src                              | 2 +-
 apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl | 2 +-
 apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl | 2 +-
 apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl | 2 +-
 apps/emqx_connector/test/emqx_connector_redis_SUITE.erl | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src
index fed9e6bc2..b7e65a042 100644
--- a/apps/emqx/src/emqx.app.src
+++ b/apps/emqx/src/emqx.app.src
@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.4"},
+    {vsn, "5.0.5"},
     {modules, []},
     {registered, []},
     {applications, [
diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
index 2ad1f5f8e..e918be84a 100644
--- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
@@ -95,7 +95,7 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
         status := StoppedStatus
     }} =
         emqx_resource:get_instance(PoolName),
-    ?assertEqual(StoppedStatus, disconnected),
+    ?assertEqual(stopped, StoppedStatus),
     ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)),
     % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
     ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
diff --git a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl
index d7f5cec63..3fd7191b9 100644
--- a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl
@@ -101,7 +101,7 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
         status := StoppedStatus
     }} =
         emqx_resource:get_instance(PoolName),
-    ?assertEqual(StoppedStatus, disconnected),
+    ?assertEqual(stopped, StoppedStatus),
     ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)),
     % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
     ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
diff --git a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl
index d99d8ab6c..9442a1810 100644
--- a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl
@@ -95,7 +95,7 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
         status := StoppedStatus
     }} =
         emqx_resource:get_instance(PoolName),
-    ?assertEqual(StoppedStatus, disconnected),
+    ?assertEqual(stopped, StoppedStatus),
     ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)),
     % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
     ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl
index 4770bbeee..a60702036 100644
--- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl
@@ -117,7 +117,7 @@ perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) ->
         status := StoppedStatus
     }} =
         emqx_resource:get_instance(PoolName),
-    ?assertEqual(StoppedStatus, disconnected),
+    ?assertEqual(stopped, StoppedStatus),
     ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)),
     % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
     ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),

From a2afdeeb48c3f7ee5c1313017fde4b6caf85281d Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 1 Aug 2022 13:06:28 +0800
Subject: [PATCH 036/232] feat: add test cases for batching query

---
 apps/emqx_resource/README.md                  |  2 +-
 ...t_resource.erl => emqx_connector_demo.erl} | 85 ++++++++++++++-----
 .../test/emqx_resource_SUITE.erl              | 47 +++++++++-
 3 files changed, 113 insertions(+), 21 deletions(-)
 rename apps/emqx_resource/test/{emqx_test_resource.erl => emqx_connector_demo.erl} (54%)

diff --git a/apps/emqx_resource/README.md b/apps/emqx_resource/README.md
index 04f3c2205..0f61df7ff 100644
--- a/apps/emqx_resource/README.md
+++ b/apps/emqx_resource/README.md
@@ -14,5 +14,5 @@ the config operations (like config validation, config dump back to files), and t
 And we put all the `specific` codes to the callback modules.
 
 See
-* `test/emqx_test_resource.erl` for a minimal `emqx_resource` implementation;
+* `test/emqx_connector_demo.erl` for a minimal `emqx_resource` implementation;
 * `test/emqx_resource_SUITE.erl` for examples of `emqx_resource` usage.
diff --git a/apps/emqx_resource/test/emqx_test_resource.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
similarity index 54%
rename from apps/emqx_resource/test/emqx_test_resource.erl
rename to apps/emqx_resource/test/emqx_connector_demo.erl
index 569579d27..e9c77e915 100644
--- a/apps/emqx_resource/test/emqx_test_resource.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqx_test_resource).
+-module(emqx_connector_demo).
 
 -include_lib("typerefl/include/types.hrl").
 
@@ -25,9 +25,12 @@
     on_start/2,
     on_stop/2,
     on_query/3,
+    on_batch_query/3,
     on_get_status/2
 ]).
 
+-export([counter_loop/1]).
+
 %% callbacks for emqx_resource config schema
 -export([roots/0]).
 
@@ -53,19 +56,19 @@ on_start(InstId, #{name := Name, stop_error := true} = Opts) ->
     {ok, Opts#{
         id => InstId,
         stop_error => true,
-        pid => spawn_dummy_process(Name, Register)
+        pid => spawn_counter_process(Name, Register)
     }};
 on_start(InstId, #{name := Name} = Opts) ->
     Register = maps:get(register, Opts, false),
     {ok, Opts#{
         id => InstId,
-        pid => spawn_dummy_process(Name, Register)
+        pid => spawn_counter_process(Name, Register)
     }};
 on_start(InstId, #{name := Name} = Opts) ->
     Register = maps:get(register, Opts, false),
     {ok, Opts#{
         id => InstId,
-        pid => spawn_dummy_process(Name, Register)
+        pid => spawn_counter_process(Name, Register)
     }}.
 
 on_stop(_InstId, #{stop_error := true}) ->
@@ -77,7 +80,44 @@ on_stop(_InstId, #{pid := Pid}) ->
 on_query(_InstId, get_state, State) ->
     {ok, State};
 on_query(_InstId, get_state_failed, State) ->
-    {error, State}.
+    {error, State};
+on_query(_InstId, {inc_counter, N}, #{pid := Pid}) ->
+    Pid ! {inc, N},
+    ok;
+on_query(_InstId, get_counter, #{pid := Pid}) ->
+    ReqRef = make_ref(),
+    From = {self(), ReqRef},
+    Pid ! {From, get},
+    receive
+        {ReqRef, Num} -> {ok, Num}
+    after 1000 ->
+        {error, timeout}
+    end.
+
+on_batch_query(InstId, BatchReq, State) ->
+    %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed.
+    case hd(BatchReq) of
+        {_From, {inc_counter, _}} ->
+            batch_inc_counter(InstId, BatchReq, State);
+        {_From, get_counter} ->
+            batch_get_counter(InstId, State)
+    end.
+
+batch_inc_counter(InstId, BatchReq, State) ->
+    TotalN = lists:foldl(
+        fun
+            ({_From, {inc_counter, N}}, Total) ->
+                Total + N;
+            ({_From, Req}, _Total) ->
+                error({mixed_requests_not_allowed, {inc_counter, Req}})
+        end,
+        0,
+        BatchReq
+    ),
+    on_query(InstId, {inc_counter, TotalN}, State).
+
+batch_get_counter(InstId, State) ->
+    on_query(InstId, get_counter, State).
 
 on_get_status(_InstId, #{health_check_error := true}) ->
     disconnected;
@@ -88,18 +128,25 @@ on_get_status(_InstId, #{pid := Pid}) ->
         false -> disconnected
     end.
 
-spawn_dummy_process(Name, Register) ->
+spawn_counter_process(Name, Register) ->
+    Pid = spawn_link(?MODULE, counter_loop, [#{counter => 0}]),
+    true = maybe_register(Name, Pid, Register),
+    Pid.
+
+counter_loop(#{counter := Num} = State) ->
+    NewState =
+        receive
+            {inc, N} ->
+                #{counter => Num + N};
+            {{FromPid, ReqRef}, get} ->
+                FromPid ! {ReqRef, Num},
+                State
+        end,
+    counter_loop(NewState).
+
+maybe_register(Name, Pid, true) ->
     ct:pal("---- Register Name: ~p", [Name]),
-    spawn(
-        fun() ->
-            true =
-                case Register of
-                    true -> register(Name, self());
-                    _ -> true
-                end,
-            Ref = make_ref(),
-            receive
-                Ref -> ok
-            end
-        end
-    ).
+    ct:pal("---- whereis(): ~p", [whereis(Name)]),
+    erlang:register(Name, Pid);
+maybe_register(_Name, _Pid, false) ->
+    true.
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 278f556ef..d05d4baf7 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -23,7 +23,7 @@
 -include("emqx_resource.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
 
--define(TEST_RESOURCE, emqx_test_resource).
+-define(TEST_RESOURCE, emqx_connector_demo).
 -define(ID, <<"id">>).
 -define(DEFAULT_RESOURCE_GROUP, <<"default">>).
 -define(RESOURCE_ERROR(REASON), {error, {resource_error, #{reason := REASON}}}).
@@ -184,6 +184,51 @@ t_query(_) ->
 
     ok = emqx_resource:remove_local(?ID).
 
+t_query_counter(_) ->
+    {ok, _} = emqx_resource:create_local(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource, register => true}
+    ),
+
+    {ok, 0} = emqx_resource:query(?ID, get_counter),
+    ok = emqx_resource:query(?ID, {inc_counter, 1}),
+    {ok, 1} = emqx_resource:query(?ID, get_counter),
+    ok = emqx_resource:query(?ID, {inc_counter, 5}),
+    {ok, 6} = emqx_resource:query(?ID, get_counter),
+
+    ok = emqx_resource:remove_local(?ID).
+
+t_batch_query_counter(_) ->
+    {ok, _} = emqx_resource:create_local(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource, register => true, batch_enabled => true}
+    ),
+
+    {ok, 0} = emqx_resource:query(?ID, get_counter),
+    Parent = self(),
+    Pids = [
+        erlang:spawn(fun() ->
+            ok = emqx_resource:query(?ID, {inc_counter, 1}),
+            Parent ! {complete, self()}
+        end)
+     || _ <- lists:seq(1, 1000)
+    ],
+    [
+        receive
+            {complete, Pid} -> ok
+        after 1000 ->
+            ct:fail({wait_for_query_timeout, Pid})
+        end
+     || Pid <- Pids
+    ],
+    {ok, 1000} = emqx_resource:query(?ID, get_counter),
+
+    ok = emqx_resource:remove_local(?ID).
+
 t_healthy_timeout(_) ->
     {ok, _} = emqx_resource:create_local(
         ?ID,

From f1419d52f1390036b1894ef494ff1f3c9f1628f2 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 2 Aug 2022 18:39:18 +0800
Subject: [PATCH 037/232] fix(resource): remove resource at the end of each
 test

---
 apps/emqx_resource/test/emqx_resource_SUITE.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index d05d4baf7..4c5a36f4b 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -36,6 +36,8 @@ groups() ->
 
 init_per_testcase(_, Config) ->
     Config.
+end_per_testcase(_, _Config) ->
+    _ = emqx_resource:remove(?ID).
 
 init_per_suite(Config) ->
     code:ensure_loaded(?TEST_RESOURCE),

From 35fe70b887e0d0f3bd9c235fabff822596a11e40 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 8 Aug 2022 17:52:44 +0800
Subject: [PATCH 038/232] feat: support aysnc callback to connector modules

---
 .../emqx_authn_jwks_connector.erl             |   3 +
 .../test/emqx_authz_api_cache_SUITE.erl       |   2 +
 .../src/emqx_connector_http.erl               |   3 +
 .../src/emqx_connector_ldap.erl               |   3 +
 .../src/emqx_connector_mongo.erl              |   3 +
 .../src/emqx_connector_mqtt.erl               |   3 +
 .../src/emqx_connector_mysql.erl              |   3 +
 .../src/emqx_connector_pgsql.erl              |   3 +
 .../src/emqx_connector_redis.erl              |   3 +
 apps/emqx_resource/include/emqx_resource.hrl  |   1 +
 apps/emqx_resource/src/emqx_resource.erl      |  11 +-
 .../src/emqx_resource_manager.erl             |   5 +-
 .../src/emqx_resource_worker.erl              | 210 +++++++++++-------
 .../test/emqx_connector_demo.erl              |   9 +-
 14 files changed, 173 insertions(+), 89 deletions(-)

diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl
index cd8451ac9..480950143 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl
@@ -22,6 +22,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -31,6 +32,8 @@
 
 -define(DEFAULT_POOL_SIZE, 8).
 
+callback_mode() -> always_sync.
+
 on_start(InstId, Opts) ->
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
     PoolOpts = [
diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
index 306fe3f13..0c49cc03a 100644
--- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
@@ -23,6 +23,8 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
+suite() -> [{timetrap, {seconds, 60}}].
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index e0d5ccfe0..c5a1b89db 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -26,6 +26,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -164,6 +165,8 @@ ref(Field) -> hoconsc:ref(?MODULE, Field).
 
 %% ===================================================================
 
+callback_mode() -> always_sync.
+
 on_start(
     InstId,
     #{
diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl
index 51d18b534..d53c0e41b 100644
--- a/apps/emqx_connector/src/emqx_connector_ldap.erl
+++ b/apps/emqx_connector/src/emqx_connector_ldap.erl
@@ -25,6 +25,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -42,6 +43,8 @@ roots() ->
 fields(_) -> [].
 
 %% ===================================================================
+callback_mode() -> always_sync.
+
 on_start(
     InstId,
     #{
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index db8b1e632..07208545f 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -25,6 +25,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -139,6 +140,8 @@ mongo_fields() ->
 
 %% ===================================================================
 
+callback_mode() -> always_sync.
+
 on_start(
     InstId,
     Config = #{
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 66682caeb..e37f6a9a2 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -24,6 +24,7 @@
 
 %% API and callbacks for supervisor
 -export([
+    callback_mode/0,
     start_link/0,
     init/1,
     create_bridge/1,
@@ -139,6 +140,8 @@ on_message_received(Msg, HookPoint, ResId) ->
     emqx:run_hook(HookPoint, [Msg]).
 
 %% ===================================================================
+callback_mode() -> always_sync.
+
 on_start(InstId, Conf) ->
     InstanceId = binary_to_atom(InstId, utf8),
     ?SLOG(info, #{
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index e818bc6ef..b379e511c 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -24,6 +24,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -73,6 +74,8 @@ server(desc) -> ?DESC("server");
 server(_) -> undefined.
 
 %% ===================================================================
+callback_mode() -> always_sync.
+
 -spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}.
 on_start(
     InstId,
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index d31c1316f..4b188e5a5 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -27,6 +27,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -66,6 +67,8 @@ server(desc) -> ?DESC("server");
 server(_) -> undefined.
 
 %% ===================================================================
+callback_mode() -> always_sync.
+
 on_start(
     InstId,
     #{
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 4826a170b..fae628d9e 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -26,6 +26,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -112,6 +113,8 @@ servers(desc) -> ?DESC("servers");
 servers(_) -> undefined.
 
 %% ===================================================================
+callback_mode() -> always_sync.
+
 on_start(
     InstId,
     #{
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 13ffff587..c691789c2 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -25,6 +25,7 @@
 -type resource_data() :: #{
     id := resource_id(),
     mod := module(),
+    callback_mode := always_sync | async_if_possible,
     config := resource_config(),
     state := resource_state(),
     status := resource_status(),
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index af047060c..a54a77e19 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -75,8 +75,7 @@
     %% stop the instance
     stop/1,
     %% query the instance
-    query/2,
-    query_async/3
+    query/2
 ]).
 
 %% Direct calls to the callback module
@@ -224,12 +223,12 @@ reset_metrics(ResId) ->
 %% =================================================================================
 -spec query(resource_id(), Request :: term()) -> Result :: term().
 query(ResId, Request) ->
-    emqx_resource_worker:query(ResId, Request).
+    query(ResId, Request, #{}).
 
--spec query_async(resource_id(), Request :: term(), emqx_resource_worker:reply_fun()) ->
+-spec query(resource_id(), Request :: term(), emqx_resource_worker:query_opts()) ->
     Result :: term().
-query_async(ResId, Request, ReplyFun) ->
-    emqx_resource_worker:query_async(ResId, Request, ReplyFun).
+query(ResId, Request, Opts) ->
+    emqx_resource_worker:query(ResId, Request, Opts).
 
 -spec start(resource_id()) -> ok | {error, Reason :: term()}.
 start(ResId) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 7dad85d5b..d6e5a1493 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -53,7 +53,7 @@
 -export([init/1, callback_mode/0, handle_event/4, terminate/3]).
 
 % State record
--record(data, {id, manager_id, group, mod, config, opts, status, state, error}).
+-record(data, {id, manager_id, group, mod, callback_mode, config, opts, status, state, error}).
 
 -define(SHORT_HEALTHCHECK_INTERVAL, 1000).
 -define(HEALTHCHECK_INTERVAL, 15000).
@@ -259,6 +259,7 @@ start_link(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         manager_id = MgrId,
         group = Group,
         mod = ResourceType,
+        callback_mode = ResourceType:callback_mode(),
         config = Config,
         opts = Opts,
         status = connecting,
@@ -559,10 +560,12 @@ maybe_reply(Actions, undefined, _Reply) ->
 maybe_reply(Actions, From, Reply) ->
     [{reply, From, Reply} | Actions].
 
+-spec data_record_to_external_map_with_metrics(#data{}) -> resource_data().
 data_record_to_external_map_with_metrics(Data) ->
     #{
         id => Data#data.id,
         mod => Data#data.mod,
+        callback_mode => Data#data.callback_mode,
         config => Data#data.config,
         status => Data#data.status,
         state => Data#data.state,
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 055fbfc53..2115fed86 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -27,11 +27,9 @@
 
 -export([
     start_link/3,
-    query/2,
     query/3,
-    query_async/3,
-    query_async/4,
     block/1,
+    block/2,
     resume/1
 ]).
 
@@ -46,6 +44,8 @@
 
 -export([queue_item_marshaller/1, estimate_size/1]).
 
+-export([reply_after_query/4, batch_reply_after_query/4]).
+
 -define(RESUME_INTERVAL, 15000).
 
 %% count
@@ -55,8 +55,8 @@
 
 -define(Q_ITEM(REQUEST), {q_item, REQUEST}).
 
--define(QUERY(FROM, REQUEST), {FROM, REQUEST}).
--define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}).
+-define(QUERY(FROM, REQUEST), {query, FROM, REQUEST}).
+-define(REPLY(FROM, REQUEST, RESULT), {reply, FROM, REQUEST, RESULT}).
 -define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]).
 
 -define(RESOURCE_ERROR(Reason, Msg),
@@ -65,10 +65,17 @@
 -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
 
 -type id() :: binary().
+-type query() :: {query, from(), request()}.
 -type request() :: term().
 -type result() :: term().
 -type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined.
 -type from() :: pid() | reply_fun().
+-type query_opts() :: #{
+    %% The key used for picking a resource worker
+    pick_key => term()
+}.
+
+-export_type([query_opts/0]).
 
 -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) ->
     {{from(), result()}, NewCbState :: term()}.
@@ -78,26 +85,20 @@ callback_mode() -> [state_functions].
 start_link(Id, Index, Opts) ->
     gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []).
 
--spec query(id(), request()) -> Result :: term().
-query(Id, Request) ->
-    query(Id, self(), Request).
-
--spec query(id(), term(), request()) -> Result :: term().
-query(Id, PickKey, Request) ->
-    pick_query(call, Id, PickKey, {query, Request}).
-
--spec query_async(id(), request(), reply_fun()) -> Result :: term().
-query_async(Id, Request, ReplyFun) ->
-    query_async(Id, self(), Request, ReplyFun).
-
--spec query_async(id(), term(), request(), reply_fun()) -> Result :: term().
-query_async(Id, PickKey, Request, ReplyFun) ->
-    pick_query(cast, Id, PickKey, {query, Request, ReplyFun}).
+-spec query(id(), request(), query_opts()) -> Result :: term().
+query(Id, Request, Opts) ->
+    PickKey = maps:get(pick_key, Opts, self()),
+    Timeout = maps:get(timeout, Opts, infinity),
+    pick_call(Id, PickKey, {query, Request}, Timeout).
 
 -spec block(pid() | atom()) -> ok.
 block(ServerRef) ->
     gen_statem:cast(ServerRef, block).
 
+-spec block(pid() | atom(), [query()]) -> ok.
+block(ServerRef, Query) ->
+    gen_statem:cast(ServerRef, {block, Query}).
+
 -spec resume(pid() | atom()) -> ok.
 resume(ServerRef) ->
     gen_statem:cast(ServerRef, resume).
@@ -121,6 +122,12 @@ init({Id, Index, Opts}) ->
     St = #{
         id => Id,
         index => Index,
+        %% query_mode = dynamic | sync | async
+        %% TODO:
+        %%  dynamic mode is async mode when things are going well, but becomes sync mode
+        %%  if the resource worker is overloaded
+        query_mode => maps:get(query_mode, Opts, sync),
+        async_reply_fun => maps:get(async_reply_fun, Opts, undefined),
         batch_enabled => maps:get(batch_enabled, Opts, false),
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
@@ -135,9 +142,11 @@ running(cast, resume, _St) ->
     keep_state_and_data;
 running(cast, block, St) ->
     {next_state, block, St};
-running(cast, {query, Request, ReplyFun}, St) ->
-    query_or_acc(ReplyFun, Request, St);
-running({call, From}, {query, Request}, St) ->
+running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) ->
+    Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]),
+    {next_state, block, St#{queue := Q1}};
+running({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) ->
+    From = maybe_quick_return(QM, From0, ReplyFun),
     query_or_acc(From, Request, St);
 running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) ->
     flush(St#{tref := undefined});
@@ -149,13 +158,15 @@ running(info, Info, _St) ->
 
 blocked(cast, block, _St) ->
     keep_state_and_data;
+blocked(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) ->
+    Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]),
+    {keep_state, St#{queue := Q1}};
 blocked(cast, resume, St) ->
     do_resume(St);
 blocked(state_timeout, resume, St) ->
     do_resume(St);
-blocked(cast, {query, Request, ReplyFun}, St) ->
-    handle_blocked(ReplyFun, Request, St);
-blocked({call, From}, {query, Request}, St) ->
+blocked({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) ->
+    From = maybe_quick_return(QM, From0, ReplyFun),
     handle_blocked(From, Request, St).
 
 terminate(_Reason, #{id := Id, index := Index}) ->
@@ -173,15 +184,23 @@ estimate_size(QItem) ->
     size(queue_item_marshaller(QItem)).
 
 %%==============================================================================
-pick_query(Fun, Id, Key, Query) ->
+maybe_quick_return(sync, From, _ReplyFun) ->
+    From;
+maybe_quick_return(async, From, ReplyFun) ->
+    ok = gen_statem:reply(From),
+    ReplyFun.
+
+pick_call(Id, Key, Query, Timeout) ->
     try gproc_pool:pick_worker(Id, Key) of
         Pid when is_pid(Pid) ->
-            gen_statem:Fun(Pid, Query);
+            gen_statem:call(Pid, Query, {clean_timeout, Timeout});
         _ ->
             ?RESOURCE_ERROR(not_created, "resource not found")
     catch
         error:badarg ->
-            ?RESOURCE_ERROR(not_created, "resource not found")
+            ?RESOURCE_ERROR(not_created, "resource not found");
+        exit:{timeout, _} ->
+            ?RESOURCE_ERROR(timeout, "call resource timeout")
     end.
 
 do_resume(#{queue := undefined} = St) ->
@@ -190,8 +209,8 @@ do_resume(#{queue := Q, id := Id} = St) ->
     case replayq:peek(Q) of
         empty ->
             {next_state, running, St};
-        ?Q_ITEM(First) ->
-            Result = call_query(Id, First),
+        ?Q_ITEM(FirstQuery) ->
+            Result = call_query(sync, Id, FirstQuery, 1),
             case handle_query_result(Id, Result, false) of
                 %% Send failed because resource down
                 true ->
@@ -206,8 +225,8 @@ do_resume(#{queue := Q, id := Id} = St) ->
 
 handle_blocked(From, Request, #{id := Id, queue := Q} = St) ->
     Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
-    _ = reply_caller(Id, ?REPLY(From, Request, Error), false),
-    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}}.
+    _ = reply_caller(Id, ?REPLY(From, Request, Error)),
+    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}}.
 
 drop_head(Q) ->
     {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
@@ -221,17 +240,18 @@ query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Lef
         true -> flush(St);
         false -> {keep_state, ensure_flush_timer(St)}
     end;
-query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id} = St) ->
-    case send_query(From, Request, Id) of
+query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, query_mode := QM} = St) ->
+    case send_query(QM, From, Request, Id) of
         true ->
-            {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}};
+            Query = ?QUERY(From, Request),
+            {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Query)])}};
         false ->
             {keep_state, St}
     end.
 
-send_query(From, Request, Id) ->
-    Result = call_query(Id, Request),
-    reply_caller(Id, ?REPLY(From, Request, Result), false).
+send_query(QM, From, Request, Id) ->
+    Result = call_query(QM, Id, ?QUERY(From, Request), 1),
+    reply_caller(Id, ?REPLY(From, Request, Result)).
 
 flush(#{acc := []} = St) ->
     {keep_state, St};
@@ -240,31 +260,37 @@ flush(
         id := Id,
         acc := Batch,
         batch_size := Size,
-        queue := Q0
+        queue := Q0,
+        query_mode := QM
     } = St
 ) ->
-    BatchResults = maybe_expand_batch_result(call_batch_query(Id, Batch), Batch),
+    Result = call_query(QM, Id, Batch, length(Batch)),
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
-    case batch_reply_caller(Id, BatchResults) of
+    case batch_reply_caller(Id, Result, Batch) of
         true ->
-            Q1 = maybe_append_queue(Q0, [?Q_ITEM(Request) || ?QUERY(_, Request) <- Batch]),
+            Q1 = maybe_append_queue(Q0, [?Q_ITEM(Query) || Query <- Batch]),
             {next_state, blocked, St1#{queue := Q1}};
         false ->
             {keep_state, St1}
     end.
 
-maybe_append_queue(undefined, _Request) -> undefined;
-maybe_append_queue(Q, Request) -> replayq:append(Q, Request).
+maybe_append_queue(undefined, _Items) -> undefined;
+maybe_append_queue(Q, Items) -> replayq:append(Q, Items).
 
-batch_reply_caller(Id, BatchResults) ->
+batch_reply_caller(Id, BatchResult, Batch) ->
     lists:foldl(
         fun(Reply, BlockWorker) ->
             reply_caller(Id, Reply, BlockWorker)
         end,
         false,
-        BatchResults
+        %% the `Mod:on_batch_query/3` returns a single result for a batch,
+        %% so we need to expand
+        ?EXPAND(BatchResult, Batch)
     ).
 
+reply_caller(Id, Reply) ->
+    reply_caller(Id, Reply, false).
+
 reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) ->
     handle_query_result(Id, Result, BlockWorker);
 reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) ->
@@ -295,45 +321,77 @@ handle_query_result(Id, Result, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, success),
     BlockWorker.
 
-call_query(Id, Request) ->
-    do_call_query(on_query, Id, Request, 1).
-
-call_batch_query(Id, Batch) ->
-    do_call_query(on_batch_query, Id, Batch, length(Batch)).
-
-do_call_query(Fun, Id, Data, Count) ->
+call_query(QM, Id, Query, QueryLen) ->
     case emqx_resource_manager:ets_lookup(Id) of
-        {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
-            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, Count),
-            try Mod:Fun(Id, Data, ResourceState) of
-                %% if the callback module (connector) wants to return an error that
-                %% makes the current resource goes into the `error` state, it should
-                %% return `{resource_down, Reason}`
-                Result -> Result
-            catch
-                Err:Reason:ST ->
-                    Msg = io_lib:format(
-                        "call query failed, func: ~s:~s/3, error: ~0p",
-                        [Mod, Fun, {Err, Reason, ST}]
-                    ),
-                    ?RESOURCE_ERROR(exception, Msg)
-            end;
+        {ok, _Group, #{callback_mode := CM, mod := Mod, state := ResSt, status := connected}} ->
+            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, QueryLen),
+            apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt);
         {ok, _Group, #{status := stopped}} ->
             ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
-        {ok, _Group, _Data} ->
+        {ok, _Group, #{status := S}} when S == connecting; S == disconnected ->
             ?RESOURCE_ERROR(not_connected, "resource not connected");
         {error, not_found} ->
             ?RESOURCE_ERROR(not_found, "resource not found")
     end.
 
-%% the result is already expaned by the `Mod:on_query/3`
-maybe_expand_batch_result(Results, _Batch) when is_list(Results) ->
-    Results;
-%% the `Mod:on_query/3` returns a sinle result for a batch, so it is need expand
-maybe_expand_batch_result(Result, Batch) ->
-    ?EXPAND(Result, Batch).
+-define(APPLY_RESOURCE(EXPR, REQ),
+    try
+        %% if the callback module (connector) wants to return an error that
+        %% makes the current resource goes into the `error` state, it should
+        %% return `{resource_down, Reason}`
+        EXPR
+    catch
+        ERR:REASON:STACKTRACE ->
+            MSG = io_lib:format(
+                "call query failed, func: ~s, id: ~s, error: ~0p, Request: ~0p",
+                [??EXPR, Id, {ERR, REASON, STACKTRACE}, REQ],
+                [{chars_limit, 1024}]
+            ),
+            ?RESOURCE_ERROR(exception, MSG)
+    end
+).
+
+apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request), ResSt) ->
+    ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request);
+apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) ->
+    ReplyFun = fun ?MODULE:reply_after_query/4,
+    ?APPLY_RESOURCE(
+        begin
+            _ = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt),
+            ok_async
+        end,
+        Request
+    );
+apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) ->
+    Requests = [Request || ?QUERY(_From, Request) <- Batch],
+    ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch);
+apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) ->
+    Requests = [Request || ?QUERY(_From, Request) <- Batch],
+    ReplyFun = fun ?MODULE:batch_reply_after_query/4,
+    ?APPLY_RESOURCE(
+        begin
+            _ = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt),
+            ok_async
+        end,
+        Batch
+    ).
+
+reply_after_query(Pid, Id, ?QUERY(From, Request) = Query, Result) ->
+    case reply_caller(Id, ?REPLY(From, Request, Result)) of
+        true -> ?MODULE:block(Pid, [Query]);
+        false -> ok
+    end.
+
+batch_reply_after_query(Pid, Id, Batch, Result) ->
+    case batch_reply_caller(Id, Result, Batch) of
+        true -> ?MODULE:block(Pid, Batch);
+        false -> ok
+    end.
 
 %%==============================================================================
+call_mode(sync, _) -> sync;
+call_mode(async, always_sync) -> sync;
+call_mode(async, async_if_possible) -> async.
 
 is_ok_result(ok) ->
     true;
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index e9c77e915..740f110ec 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -22,6 +22,7 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -49,6 +50,8 @@ register(required) -> true;
 register(default) -> false;
 register(_) -> undefined.
 
+callback_mode() -> always_sync.
+
 on_start(_InstId, #{create_error := true}) ->
     error("some error");
 on_start(InstId, #{name := Name, stop_error := true} = Opts) ->
@@ -58,12 +61,6 @@ on_start(InstId, #{name := Name, stop_error := true} = Opts) ->
         stop_error => true,
         pid => spawn_counter_process(Name, Register)
     }};
-on_start(InstId, #{name := Name} = Opts) ->
-    Register = maps:get(register, Opts, false),
-    {ok, Opts#{
-        id => InstId,
-        pid => spawn_counter_process(Name, Register)
-    }};
 on_start(InstId, #{name := Name} = Opts) ->
     Register = maps:get(register, Opts, false),
     {ok, Opts#{

From 145ff66a9add21898ee6f6805273d72cb57ab58d Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 9 Aug 2022 08:58:42 +0800
Subject: [PATCH 039/232] fix: issues found by dialyzer and elvis

---
 apps/emqx_resource/include/emqx_resource.hrl  |  3 +-
 apps/emqx_resource/src/emqx_resource.erl      |  8 ++++-
 .../src/emqx_resource_manager.erl             |  5 ++--
 .../src/emqx_ee_connector_hstreamdb.erl       | 29 +++++++++----------
 .../src/emqx_ee_connector_influxdb.erl        | 17 ++++++-----
 5 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index c691789c2..a59877a30 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -22,10 +22,11 @@
 -type resource_spec() :: map().
 -type resource_state() :: term().
 -type resource_status() :: connected | disconnected | connecting | stopped.
+-type callback_mode() :: always_sync | async_if_possible.
 -type resource_data() :: #{
     id := resource_id(),
     mod := module(),
-    callback_mode := always_sync | async_if_possible,
+    callback_mode := callback_mode(),
     config := resource_config(),
     state := resource_state(),
     status := resource_status(),
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index a54a77e19..d17f4ce19 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -80,8 +80,10 @@
 
 %% Direct calls to the callback module
 
-%% start the instance
 -export([
+    %% get the callback mode of a specific module
+    get_callback_mode/1,
+    %% start the instance
     call_start/3,
     %% verify if the resource is working normally
     call_health_check/3,
@@ -285,6 +287,10 @@ generate_id(Name) when is_binary(Name) ->
 -spec list_group_instances(resource_group()) -> [resource_id()].
 list_group_instances(Group) -> emqx_resource_manager:list_group(Group).
 
+-spec get_callback_mode(module()) -> callback_mode().
+get_callback_mode(Mod) ->
+    Mod:callback_mode().
+
 -spec call_start(manager_id(), module(), resource_config()) ->
     {ok, resource_state()} | {error, Reason :: term()}.
 call_start(MgrId, Mod, Config) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index d6e5a1493..3310555d1 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -54,6 +54,7 @@
 
 % State record
 -record(data, {id, manager_id, group, mod, callback_mode, config, opts, status, state, error}).
+-type data() :: #data{}.
 
 -define(SHORT_HEALTHCHECK_INTERVAL, 1000).
 -define(HEALTHCHECK_INTERVAL, 15000).
@@ -259,7 +260,7 @@ start_link(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         manager_id = MgrId,
         group = Group,
         mod = ResourceType,
-        callback_mode = ResourceType:callback_mode(),
+        callback_mode = emqx_resource:get_callback_mode(ResourceType),
         config = Config,
         opts = Opts,
         status = connecting,
@@ -560,7 +561,7 @@ maybe_reply(Actions, undefined, _Reply) ->
 maybe_reply(Actions, From, Reply) ->
     [{reply, From, Reply} | Actions].
 
--spec data_record_to_external_map_with_metrics(#data{}) -> resource_data().
+-spec data_record_to_external_map_with_metrics(data()) -> resource_data().
 data_record_to_external_map_with_metrics(Data) ->
     #{
         id => Data#data.id,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
index 8ee37cd8a..3892b7fc0 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
@@ -13,9 +13,10 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -33,6 +34,7 @@
 
 %% -------------------------------------------------------------------------------------------------
 %% resource callback
+callback_mode() -> always_sync.
 
 on_start(InstId, Config) ->
     start_client(InstId, Config).
@@ -52,11 +54,10 @@ on_stop(InstId, #{client := Client, producer := Producer}) ->
 on_query(
     _InstId,
     {send_message, Data},
-    AfterQuery,
     #{producer := Producer, ordering_key := OrderingKey, payload := Payload}
 ) ->
     Record = to_record(OrderingKey, Payload, Data),
-    do_append(AfterQuery, Producer, Record).
+    do_append(Producer, Record).
 
 on_get_status(_InstId, #{client := Client}) ->
     case is_alive(Client) of
@@ -260,27 +261,26 @@ to_record(OrderingKey, Payload) when is_binary(OrderingKey) ->
 to_record(OrderingKey, Payload) ->
     hstreamdb:to_record(OrderingKey, raw, Payload).
 
-do_append(AfterQuery, Producer, Record) ->
-    do_append(AfterQuery, false, Producer, Record).
+do_append(Producer, Record) ->
+    do_append(false, Producer, Record).
 
 %% TODO: this append is async, remove or change it after we have better disk cache.
-% do_append(AfterQuery, true, Producer, Record) ->
+% do_append(true, Producer, Record) ->
 %     case hstreamdb:append(Producer, Record) of
 %         ok ->
 %             ?SLOG(debug, #{
 %                 msg => "hstreamdb producer async append success",
 %                 record => Record
-%             }),
-%             emqx_resource:query_success(AfterQuery);
-%         {error, Reason} ->
+%             });
+%         {error, Reason} = Err ->
 %             ?SLOG(error, #{
 %                 msg => "hstreamdb producer async append failed",
 %                 reason => Reason,
 %                 record => Record
 %             }),
-%             emqx_resource:query_failed(AfterQuery)
+%             Err
 %     end;
-do_append(AfterQuery, false, Producer, Record) ->
+do_append(false, Producer, Record) ->
     %% TODO: this append is sync, but it does not support [Record], can only append one Record.
     %% Change it after we have better dick cache.
     case hstreamdb:append_flush(Producer, Record) of
@@ -288,15 +288,14 @@ do_append(AfterQuery, false, Producer, Record) ->
             ?SLOG(debug, #{
                 msg => "hstreamdb producer sync append success",
                 record => Record
-            }),
-            emqx_resource:query_success(AfterQuery);
-        {error, Reason} ->
+            });
+        {error, Reason} = Err ->
             ?SLOG(error, #{
                 msg => "hstreamdb producer sync append failed",
                 reason => Reason,
                 record => Record
             }),
-            emqx_resource:query_failed(AfterQuery)
+            Err
     end.
 
 client_name(InstId) ->
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 9582f1729..09b3d7350 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -13,9 +13,10 @@
 
 %% callbacks of behaviour emqx_resource
 -export([
+    callback_mode/0,
     on_start/2,
     on_stop/2,
-    on_query/4,
+    on_query/3,
     on_get_status/2
 ]).
 
@@ -28,6 +29,7 @@
 
 %% -------------------------------------------------------------------------------------------------
 %% resource callback
+callback_mode() -> always_sync.
 
 on_start(InstId, Config) ->
     start_client(InstId, Config).
@@ -35,8 +37,8 @@ on_start(InstId, Config) ->
 on_stop(_InstId, #{client := Client}) ->
     influxdb:stop_client(Client).
 
-on_query(InstId, {send_message, Data}, AfterQuery, State) ->
-    do_query(InstId, {send_message, Data}, AfterQuery, State).
+on_query(InstId, {send_message, Data}, State) ->
+    do_query(InstId, {send_message, Data}, State).
 
 on_get_status(_InstId, #{client := Client}) ->
     case influxdb:is_alive(Client) of
@@ -308,7 +310,7 @@ ssl_config(SSL = #{enable := true}) ->
 %% -------------------------------------------------------------------------------------------------
 %% Query
 
-do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client}) ->
+do_query(InstId, {send_message, Data}, State = #{client := Client}) ->
     {Points, Errs} = data_to_points(Data, State),
     lists:foreach(
         fun({error, Reason}) ->
@@ -326,15 +328,14 @@ do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client})
                 msg => "influxdb write point success",
                 connector => InstId,
                 points => Points
-            }),
-            emqx_resource:query_success(AfterQuery);
-        {error, Reason} ->
+            });
+        {error, Reason} = Err ->
             ?SLOG(error, #{
                 msg => "influxdb write point failed",
                 connector => InstId,
                 reason => Reason
             }),
-            emqx_resource:query_failed(AfterQuery)
+            Err
     end.
 
 %% -------------------------------------------------------------------------------------------------

From efd6c56dd9554a05b8b0772856008cdae40ab6ad Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 9 Aug 2022 13:00:21 +0800
Subject: [PATCH 040/232] fix: test cases for batch query sync

---
 .../src/emqx_resource_worker.erl              |  7 +++-
 .../test/emqx_connector_demo.erl              |  8 ++---
 .../test/emqx_resource_SUITE.erl              | 35 +++++++++++++++----
 3 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 2115fed86..e20345c2b 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -22,6 +22,7 @@
 -include("emqx_resource.hrl").
 -include("emqx_resource_utils.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -behaviour(gen_statem).
 
@@ -351,9 +352,11 @@ call_query(QM, Id, Query, QueryLen) ->
     end
 ).
 
-apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request), ResSt) ->
+apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request) = _Query, ResSt) ->
+    ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
     ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request);
 apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) ->
+    ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
     ReplyFun = fun ?MODULE:reply_after_query/4,
     ?APPLY_RESOURCE(
         begin
@@ -363,9 +366,11 @@ apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) ->
         Request
     );
 apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) ->
+    ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
     ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch);
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) ->
+    ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
     ReplyFun = fun ?MODULE:batch_reply_after_query/4,
     ?APPLY_RESOURCE(
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index 740f110ec..40734de68 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -94,18 +94,18 @@ on_query(_InstId, get_counter, #{pid := Pid}) ->
 on_batch_query(InstId, BatchReq, State) ->
     %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed.
     case hd(BatchReq) of
-        {_From, {inc_counter, _}} ->
+        {inc_counter, _} ->
             batch_inc_counter(InstId, BatchReq, State);
-        {_From, get_counter} ->
+        get_counter ->
             batch_get_counter(InstId, State)
     end.
 
 batch_inc_counter(InstId, BatchReq, State) ->
     TotalN = lists:foldl(
         fun
-            ({_From, {inc_counter, N}}, Total) ->
+            ({inc_counter, N}, Total) ->
                 Total + N;
-            ({_From, Req}, _Total) ->
+            (Req, _Total) ->
                 error({mixed_requests_not_allowed, {inc_counter, Req}})
         end,
         0,
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 4c5a36f4b..fb2bdfd7c 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -22,6 +22,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include("emqx_resource.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(TEST_RESOURCE, emqx_connector_demo).
 -define(ID, <<"id">>).
@@ -207,17 +208,40 @@ t_batch_query_counter(_) ->
         ?ID,
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
-        #{name => test_resource, register => true, batch_enabled => true}
+        #{name => test_resource, register => true},
+        #{batch_enabled => true}
     ),
 
-    {ok, 0} = emqx_resource:query(?ID, get_counter),
+    ?check_trace(
+        #{timetrap => 10000, timeout => 1000},
+        emqx_resource:query(?ID, get_counter),
+        fun(Result, Trace) ->
+            ?assertMatch({ok, 0}, Result),
+            QueryTrace = ?of_kind(call_batch_query, Trace),
+            ?assertMatch([#{batch := [{query, _, get_counter}]}], QueryTrace)
+        end
+    ),
+
+    ?check_trace(
+        #{timetrap => 10000, timeout => 1000},
+        inc_counter_in_parallel(1000),
+        fun(Trace) ->
+            QueryTrace = ?of_kind(call_batch_query, Trace),
+            ?assertMatch([#{batch := BatchReq} | _] when length(BatchReq) > 1, QueryTrace)
+        end
+    ),
+    {ok, 1000} = emqx_resource:query(?ID, get_counter),
+
+    ok = emqx_resource:remove_local(?ID).
+
+inc_counter_in_parallel(N) ->
     Parent = self(),
     Pids = [
         erlang:spawn(fun() ->
             ok = emqx_resource:query(?ID, {inc_counter, 1}),
             Parent ! {complete, self()}
         end)
-     || _ <- lists:seq(1, 1000)
+     || _ <- lists:seq(1, N)
     ],
     [
         receive
@@ -226,10 +250,7 @@ t_batch_query_counter(_) ->
             ct:fail({wait_for_query_timeout, Pid})
         end
      || Pid <- Pids
-    ],
-    {ok, 1000} = emqx_resource:query(?ID, get_counter),
-
-    ok = emqx_resource:remove_local(?ID).
+    ].
 
 t_healthy_timeout(_) ->
     {ok, _} = emqx_resource:create_local(

From 82550a585a4a0ce7a3e72733d67d683c03f00645 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 10 Aug 2022 00:30:42 +0800
Subject: [PATCH 041/232] fix: add test cases for query async

---
 apps/emqx_resource/include/emqx_resource.hrl  |   6 +
 apps/emqx_resource/src/emqx_resource.erl      |  13 +-
 .../src/emqx_resource_worker.erl              |  48 +++++--
 .../test/emqx_connector_demo.erl              |  17 ++-
 .../test/emqx_resource_SUITE.erl              | 122 +++++++++++++++---
 5 files changed, 171 insertions(+), 35 deletions(-)

diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index a59877a30..75cba14ad 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -23,6 +23,12 @@
 -type resource_state() :: term().
 -type resource_status() :: connected | disconnected | connecting | stopped.
 -type callback_mode() :: always_sync | async_if_possible.
+-type result() :: term().
+-type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined.
+-type query_opts() :: #{
+    %% The key used for picking a resource worker
+    pick_key => term()
+}.
 -type resource_data() :: #{
     id := resource_id(),
     mod := module(),
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index d17f4ce19..f3f2d5fb9 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -75,7 +75,10 @@
     %% stop the instance
     stop/1,
     %% query the instance
-    query/2
+    query/2,
+    %% query the instance without batching and queuing messages.
+    simple_sync_query/2,
+    simple_async_query/3
 ]).
 
 %% Direct calls to the callback module
@@ -232,6 +235,14 @@ query(ResId, Request) ->
 query(ResId, Request, Opts) ->
     emqx_resource_worker:query(ResId, Request, Opts).
 
+-spec simple_sync_query(resource_id(), Request :: term()) -> Result :: term().
+simple_sync_query(ResId, Request) ->
+    emqx_resource_worker:simple_sync_query(ResId, Request).
+
+-spec simple_async_query(resource_id(), Request :: term(), reply_fun()) -> Result :: term().
+simple_async_query(ResId, Request, ReplyFun) ->
+    emqx_resource_worker:simple_async_query(ResId, Request, ReplyFun).
+
 -spec start(resource_id()) -> ok | {error, Reason :: term()}.
 start(ResId) ->
     start(ResId, #{}).
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index e20345c2b..d19353a29 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -34,6 +34,11 @@
     resume/1
 ]).
 
+-export([
+    simple_sync_query/2,
+    simple_async_query/3
+]).
+
 -export([
     callback_mode/0,
     init/1,
@@ -68,13 +73,7 @@
 -type id() :: binary().
 -type query() :: {query, from(), request()}.
 -type request() :: term().
--type result() :: term().
--type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined.
 -type from() :: pid() | reply_fun().
--type query_opts() :: #{
-    %% The key used for picking a resource worker
-    pick_key => term()
-}.
 
 -export_type([query_opts/0]).
 
@@ -92,6 +91,19 @@ query(Id, Request, Opts) ->
     Timeout = maps:get(timeout, Opts, infinity),
     pick_call(Id, PickKey, {query, Request}, Timeout).
 
+%% simple query the resource without batching and queuing messages.
+-spec simple_sync_query(id(), request()) -> Result :: term().
+simple_sync_query(Id, Request) ->
+    Result = call_query(sync, Id, ?QUERY(self(), Request), 1),
+    _ = handle_query_result(Id, Result, false),
+    Result.
+
+-spec simple_async_query(id(), request(), reply_fun()) -> Result :: term().
+simple_async_query(Id, Request, ReplyFun) ->
+    Result = call_query(async, Id, ?QUERY(ReplyFun, Request), 1),
+    _ = handle_query_result(Id, Result, false),
+    Result.
+
 -spec block(pid() | atom()) -> ok.
 block(ServerRef) ->
     gen_statem:cast(ServerRef, block).
@@ -188,7 +200,7 @@ estimate_size(QItem) ->
 maybe_quick_return(sync, From, _ReplyFun) ->
     From;
 maybe_quick_return(async, From, ReplyFun) ->
-    ok = gen_statem:reply(From),
+    gen_statem:reply(From, ok),
     ReplyFun.
 
 pick_call(Id, Key, Query, Timeout) ->
@@ -295,7 +307,11 @@ reply_caller(Id, Reply) ->
 reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) ->
     handle_query_result(Id, Result, BlockWorker);
 reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) ->
-    ?SAFE_CALL(ReplyFun(Result, Args)),
+    _ =
+        case Result of
+            {async_return, _} -> ok;
+            _ -> apply(ReplyFun, Args ++ [Result])
+        end,
     handle_query_result(Id, Result, BlockWorker);
 reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
     gen_statem:reply(From, Result),
@@ -316,6 +332,10 @@ handle_query_result(Id, {error, _}, BlockWorker) ->
 handle_query_result(Id, {resource_down, _}, _BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down),
     true;
+handle_query_result(_Id, {async_return, {resource_down, _}}, _BlockWorker) ->
+    true;
+handle_query_result(_Id, {async_return, ok}, BlockWorker) ->
+    BlockWorker;
 handle_query_result(Id, Result, BlockWorker) ->
     %% assert
     true = is_ok_result(Result),
@@ -352,16 +372,16 @@ call_query(QM, Id, Query, QueryLen) ->
     end
 ).
 
-apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request) = _Query, ResSt) ->
+apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
     ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request);
-apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) ->
+apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
     ReplyFun = fun ?MODULE:reply_after_query/4,
     ?APPLY_RESOURCE(
         begin
-            _ = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt),
-            ok_async
+            Result = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt),
+            {async_return, Result}
         end,
         Request
     );
@@ -375,8 +395,8 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) ->
     ReplyFun = fun ?MODULE:batch_reply_after_query/4,
     ?APPLY_RESOURCE(
         begin
-            _ = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt),
-            ok_async
+            Result = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt),
+            {async_return, Result}
         end,
         Batch
     ).
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index 40734de68..3bea71993 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -26,11 +26,12 @@
     on_start/2,
     on_stop/2,
     on_query/3,
+    on_query_async/4,
     on_batch_query/3,
     on_get_status/2
 ]).
 
--export([counter_loop/1]).
+-export([counter_loop/1, set_callback_mode/1]).
 
 %% callbacks for emqx_resource config schema
 -export([roots/0]).
@@ -50,7 +51,12 @@ register(required) -> true;
 register(default) -> false;
 register(_) -> undefined.
 
-callback_mode() -> always_sync.
+-define(CM_KEY, {?MODULE, callback_mode}).
+callback_mode() ->
+    persistent_term:get(?CM_KEY).
+
+set_callback_mode(Mode) ->
+    persistent_term:put(?CM_KEY, Mode).
 
 on_start(_InstId, #{create_error := true}) ->
     error("some error");
@@ -91,6 +97,10 @@ on_query(_InstId, get_counter, #{pid := Pid}) ->
         {error, timeout}
     end.
 
+on_query_async(_InstId, Query, ReplyFun, State) ->
+    Result = on_query(_InstId, Query, State),
+    apply_reply(ReplyFun, Result).
+
 on_batch_query(InstId, BatchReq, State) ->
     %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed.
     case hd(BatchReq) of
@@ -147,3 +157,6 @@ maybe_register(Name, Pid, true) ->
     erlang:register(Name, Pid);
 maybe_register(_Name, _Pid, false) ->
     true.
+
+apply_reply({ReplyFun, Args}, Result) when is_function(ReplyFun) ->
+    apply(ReplyFun, Args ++ [Result]).
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index fb2bdfd7c..5177e792c 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -28,6 +28,7 @@
 -define(ID, <<"id">>).
 -define(DEFAULT_RESOURCE_GROUP, <<"default">>).
 -define(RESOURCE_ERROR(REASON), {error, {resource_error, #{reason := REASON}}}).
+-define(TRACE_OPTS, #{timetrap => 10000, timeout => 1000}).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -36,6 +37,7 @@ groups() ->
     [].
 
 init_per_testcase(_, Config) ->
+    emqx_connector_demo:set_callback_mode(always_sync),
     Config.
 end_per_testcase(_, _Config) ->
     _ = emqx_resource:remove(?ID).
@@ -213,7 +215,7 @@ t_batch_query_counter(_) ->
     ),
 
     ?check_trace(
-        #{timetrap => 10000, timeout => 1000},
+        ?TRACE_OPTS,
         emqx_resource:query(?ID, get_counter),
         fun(Result, Trace) ->
             ?assertMatch({ok, 0}, Result),
@@ -223,7 +225,7 @@ t_batch_query_counter(_) ->
     ),
 
     ?check_trace(
-        #{timetrap => 10000, timeout => 1000},
+        ?TRACE_OPTS,
         inc_counter_in_parallel(1000),
         fun(Trace) ->
             QueryTrace = ?of_kind(call_batch_query, Trace),
@@ -234,23 +236,90 @@ t_batch_query_counter(_) ->
 
     ok = emqx_resource:remove_local(?ID).
 
-inc_counter_in_parallel(N) ->
-    Parent = self(),
-    Pids = [
-        erlang:spawn(fun() ->
-            ok = emqx_resource:query(?ID, {inc_counter, 1}),
-            Parent ! {complete, self()}
-        end)
-     || _ <- lists:seq(1, N)
-    ],
-    [
-        receive
-            {complete, Pid} -> ok
-        after 1000 ->
-            ct:fail({wait_for_query_timeout, Pid})
+t_query_counter_async(_) ->
+    {ok, _} = emqx_resource:create_local(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource, register => true},
+        #{query_mode => async}
+    ),
+    ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)),
+    ?check_trace(
+        ?TRACE_OPTS,
+        inc_counter_in_parallel(1000),
+        fun(Trace) ->
+            %% the callback_mode if 'emqx_connector_demo' is 'always_sync'.
+            QueryTrace = ?of_kind(call_query, Trace),
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
         end
-     || Pid <- Pids
-    ].
+    ),
+    %% wait for 1s to make sure all the aysnc query is sent to the resource.
+    timer:sleep(1000),
+    %% simple query ignores the query_mode and batching settings in the resource_worker
+    ?check_trace(
+        ?TRACE_OPTS,
+        emqx_resource:simple_sync_query(?ID, get_counter),
+        fun(Result, Trace) ->
+            ?assertMatch({ok, 1000}, Result),
+            %% the callback_mode if 'emqx_connector_demo' is 'always_sync'.
+            QueryTrace = ?of_kind(call_query, Trace),
+            ?assertMatch([#{query := {query, _, get_counter}}], QueryTrace)
+        end
+    ),
+    {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
+    ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C),
+    ok = emqx_resource:remove_local(?ID).
+
+t_query_counter_async_2(_) ->
+    emqx_connector_demo:set_callback_mode(async_if_possible),
+
+    Tab0 = ets:new(?FUNCTION_NAME, [bag, public]),
+    Insert = fun(Tab, Result) ->
+        ets:insert(Tab, {make_ref(), Result})
+    end,
+    {ok, _} = emqx_resource:create_local(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource, register => true},
+        #{query_mode => async, async_reply_fun => {Insert, [Tab0]}}
+    ),
+    ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)),
+    ?check_trace(
+        ?TRACE_OPTS,
+        inc_counter_in_parallel(1000),
+        fun(Trace) ->
+            QueryTrace = ?of_kind(call_query_async, Trace),
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+        end
+    ),
+
+    %% wait for 1s to make sure all the aysnc query is sent to the resource.
+    timer:sleep(1000),
+    %% simple query ignores the query_mode and batching settings in the resource_worker
+    ?check_trace(
+        ?TRACE_OPTS,
+        emqx_resource:simple_sync_query(?ID, get_counter),
+        fun(Result, Trace) ->
+            ?assertMatch({ok, 1000}, Result),
+            QueryTrace = ?of_kind(call_query, Trace),
+            ?assertMatch([#{query := {query, _, get_counter}}], QueryTrace)
+        end
+    ),
+    {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
+    ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C),
+    ?assertMatch(1000, ets:info(Tab0, size)),
+    ?assert(
+        lists:all(
+            fun
+                ({_, ok}) -> true;
+                (_) -> false
+            end,
+            ets:tab2list(Tab0)
+        )
+    ),
+    ok = emqx_resource:remove_local(?ID).
 
 t_healthy_timeout(_) ->
     {ok, _} = emqx_resource:create_local(
@@ -480,6 +549,23 @@ t_auto_retry(_) ->
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------
+inc_counter_in_parallel(N) ->
+    Parent = self(),
+    Pids = [
+        erlang:spawn(fun() ->
+            emqx_resource:query(?ID, {inc_counter, 1}),
+            Parent ! {complete, self()}
+        end)
+     || _ <- lists:seq(1, N)
+    ],
+    [
+        receive
+            {complete, Pid} -> ok
+        after 1000 ->
+            ct:fail({wait_for_query_timeout, Pid})
+        end
+     || Pid <- Pids
+    ].
 
 bin_config() ->
     <<"\"name\": \"test_resource\"">>.

From 07933e36389587d57a656c10a6fee70667388c0e Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Wed, 10 Aug 2022 17:39:24 +0800
Subject: [PATCH 042/232] feat: translate map to object in schema.json

---
 apps/emqx_conf/src/emqx_conf.app.src | 2 +-
 apps/emqx_conf/src/emqx_conf.erl     | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src
index 1441a4180..854fdac07 100644
--- a/apps/emqx_conf/src/emqx_conf.app.src
+++ b/apps/emqx_conf/src/emqx_conf.app.src
@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib]},
diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl
index bc8b52702..ce54695a5 100644
--- a/apps/emqx_conf/src/emqx_conf.erl
+++ b/apps/emqx_conf/src/emqx_conf.erl
@@ -399,6 +399,10 @@ typename_to_spec("failure_strategy()", _Mod) ->
     #{type => enum, symbols => [force, drop, throw]};
 typename_to_spec("initial()", _Mod) ->
     #{type => string};
+typename_to_spec("map()", _Mod) ->
+    #{type => object};
+typename_to_spec("#{" ++ _, Mod) ->
+    typename_to_spec("map()", Mod);
 typename_to_spec(Name, Mod) ->
     Spec = range(Name),
     Spec1 = remote_module_type(Spec, Name, Mod),

From 6203a01320251d5fd93c938ccfab64cb01988b86 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 11 Aug 2022 00:29:50 +0800
Subject: [PATCH 043/232] feat: add inflight window to emqx_resource

---
 apps/emqx_bridge/src/emqx_bridge.app.src      |   2 +-
 apps/emqx_resource/include/emqx_resource.hrl  |   3 +-
 apps/emqx_resource/src/emqx_resource.erl      |   1 +
 .../src/emqx_resource_worker.erl              | 224 +++++++++++++-----
 .../test/emqx_connector_demo.erl              |  56 ++++-
 .../test/emqx_resource_SUITE.erl              | 136 ++++++++++-
 6 files changed, 341 insertions(+), 81 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src
index fe19ed066..3be8d38fd 100644
--- a/apps/emqx_bridge/src/emqx_bridge.app.src
+++ b/apps/emqx_bridge/src/emqx_bridge.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "An OTP application"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 75cba14ad..5c561a8d3 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -27,7 +27,8 @@
 -type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined.
 -type query_opts() :: #{
     %% The key used for picking a resource worker
-    pick_key => term()
+    pick_key => term(),
+    async_reply_fun => reply_fun()
 }.
 -type resource_data() :: #{
     id := resource_id(),
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index f3f2d5fb9..0d2289696 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -76,6 +76,7 @@
     stop/1,
     %% query the instance
     query/2,
+    query/3,
     %% query the instance without batching and queuing messages.
     simple_sync_query/2,
     simple_async_query/3
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index d19353a29..716842bf9 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -50,7 +50,7 @@
 
 -export([queue_item_marshaller/1, estimate_size/1]).
 
--export([reply_after_query/4, batch_reply_after_query/4]).
+-export([reply_after_query/6, batch_reply_after_query/6]).
 
 -define(RESUME_INTERVAL, 15000).
 
@@ -69,18 +69,18 @@
     {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}}
 ).
 -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
+-define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
+-define(DEFAULT_INFLIGHT, 100).
 
 -type id() :: binary().
 -type query() :: {query, from(), request()}.
 -type request() :: term().
 -type from() :: pid() | reply_fun().
 
--export_type([query_opts/0]).
-
 -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) ->
     {{from(), result()}, NewCbState :: term()}.
 
-callback_mode() -> [state_functions].
+callback_mode() -> [state_functions, state_enter].
 
 start_link(Id, Index, Opts) ->
     gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []).
@@ -89,18 +89,18 @@ start_link(Id, Index, Opts) ->
 query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
     Timeout = maps:get(timeout, Opts, infinity),
-    pick_call(Id, PickKey, {query, Request}, Timeout).
+    pick_call(Id, PickKey, {query, Request, Opts}, Timeout).
 
 %% simple query the resource without batching and queuing messages.
 -spec simple_sync_query(id(), request()) -> Result :: term().
 simple_sync_query(Id, Request) ->
-    Result = call_query(sync, Id, ?QUERY(self(), Request), 1),
+    Result = call_query(sync, Id, ?QUERY(self(), Request), #{}),
     _ = handle_query_result(Id, Result, false),
     Result.
 
 -spec simple_async_query(id(), request(), reply_fun()) -> Result :: term().
 simple_async_query(Id, Request, ReplyFun) ->
-    Result = call_query(async, Id, ?QUERY(ReplyFun, Request), 1),
+    Result = call_query(async, Id, ?QUERY(ReplyFun, Request), #{}),
     _ = handle_query_result(Id, Result, false),
     Result.
 
@@ -119,38 +119,44 @@ resume(ServerRef) ->
 init({Id, Index, Opts}) ->
     process_flag(trap_exit, true),
     true = gproc_pool:connect_worker(Id, {Id, Index}),
+    Name = name(Id, Index),
     BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE),
     Queue =
         case maps:get(queue_enabled, Opts, false) of
             true ->
                 replayq:open(#{
                     dir => disk_queue_dir(Id, Index),
-                    seg_bytes => 10000000,
+                    seg_bytes => maps:get(queue_max_bytes, Opts, ?DEFAULT_QUEUE_SIZE),
                     sizer => fun ?MODULE:estimate_size/1,
                     marshaller => fun ?MODULE:queue_item_marshaller/1
                 });
             false ->
                 undefined
         end,
+    ok = inflight_new(Name),
     St = #{
         id => Id,
         index => Index,
+        name => Name,
         %% query_mode = dynamic | sync | async
         %% TODO:
         %%  dynamic mode is async mode when things are going well, but becomes sync mode
         %%  if the resource worker is overloaded
         query_mode => maps:get(query_mode, Opts, sync),
-        async_reply_fun => maps:get(async_reply_fun, Opts, undefined),
+        async_inflight_window => maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT),
         batch_enabled => maps:get(batch_enabled, Opts, false),
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
         queue => Queue,
+        resume_interval => maps:get(resume_interval, Opts, ?RESUME_INTERVAL),
         acc => [],
         acc_left => BatchSize,
         tref => undefined
     },
     {ok, blocked, St, {next_event, cast, resume}}.
 
+running(enter, _, _St) ->
+    keep_state_and_data;
 running(cast, resume, _St) ->
     keep_state_and_data;
 running(cast, block, St) ->
@@ -158,8 +164,8 @@ running(cast, block, St) ->
 running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) ->
     Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]),
     {next_state, block, St#{queue := Q1}};
-running({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) ->
-    From = maybe_quick_return(QM, From0, ReplyFun),
+running({call, From0}, {query, Request, Opts}, #{query_mode := QM} = St) ->
+    From = maybe_quick_return(QM, From0, maps:get(async_reply_fun, Opts, undefined)),
     query_or_acc(From, Request, St);
 running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) ->
     flush(St#{tref := undefined});
@@ -169,6 +175,8 @@ running(info, Info, _St) ->
     ?SLOG(error, #{msg => unexpected_msg, info => Info}),
     keep_state_and_data.
 
+blocked(enter, _, #{resume_interval := ResumeT} = _St) ->
+    {keep_state_and_data, {state_timeout, ResumeT, resume}};
 blocked(cast, block, _St) ->
     keep_state_and_data;
 blocked(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) ->
@@ -178,9 +186,11 @@ blocked(cast, resume, St) ->
     do_resume(St);
 blocked(state_timeout, resume, St) ->
     do_resume(St);
-blocked({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) ->
-    From = maybe_quick_return(QM, From0, ReplyFun),
-    handle_blocked(From, Request, St).
+blocked({call, From0}, {query, Request, Opts}, #{id := Id, queue := Q, query_mode := QM} = St) ->
+    From = maybe_quick_return(QM, From0, maps:get(async_reply_fun, Opts, undefined)),
+    Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
+    _ = reply_caller(Id, ?REPLY(From, Request, Error)),
+    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}}.
 
 terminate(_Reason, #{id := Id, index := Index}) ->
     gproc_pool:disconnect_worker(Id, {Id, Index}).
@@ -216,30 +226,44 @@ pick_call(Id, Key, Query, Timeout) ->
             ?RESOURCE_ERROR(timeout, "call resource timeout")
     end.
 
-do_resume(#{queue := undefined} = St) ->
+do_resume(#{queue := Q, id := Id, name := Name} = St) ->
+    case inflight_get_first(Name) of
+        empty ->
+            retry_first_from_queue(Q, Id, St);
+        {Ref, FirstQuery} ->
+            retry_first_sync(Id, FirstQuery, Name, Ref, undefined, St)
+    end.
+
+retry_first_from_queue(undefined, _Id, St) ->
     {next_state, running, St};
-do_resume(#{queue := Q, id := Id} = St) ->
+retry_first_from_queue(Q, Id, St) ->
     case replayq:peek(Q) of
         empty ->
             {next_state, running, St};
         ?Q_ITEM(FirstQuery) ->
-            Result = call_query(sync, Id, FirstQuery, 1),
-            case handle_query_result(Id, Result, false) of
-                %% Send failed because resource down
-                true ->
-                    {keep_state, St, {state_timeout, ?RESUME_INTERVAL, resume}};
-                %% Send ok or failed but the resource is working
-                false ->
-                    %% We Send 'resume' to the end of the mailbox to give the worker
-                    %% a chance to process 'query' requests.
-                    {keep_state, St#{queue => drop_head(Q)}, {state_timeout, 0, resume}}
-            end
+            retry_first_sync(Id, FirstQuery, undefined, undefined, Q, St)
     end.
 
-handle_blocked(From, Request, #{id := Id, queue := Q} = St) ->
-    Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
-    _ = reply_caller(Id, ?REPLY(From, Request, Error)),
-    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}}.
+retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = St0) ->
+    Result = call_query(sync, Id, FirstQuery, #{}),
+    case handle_query_result(Id, Result, false) of
+        %% Send failed because resource down
+        true ->
+            {keep_state, St0, {state_timeout, ResumeT, resume}};
+        %% Send ok or failed but the resource is working
+        false ->
+            %% We Send 'resume' to the end of the mailbox to give the worker
+            %% a chance to process 'query' requests.
+            St =
+                case Q of
+                    undefined ->
+                        inflight_drop(Name, Ref),
+                        St0;
+                    _ ->
+                        St0#{queue => drop_head(Q)}
+                end,
+            {keep_state, St, {state_timeout, 0, resume}}
+    end.
 
 drop_head(Q) ->
     {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
@@ -254,7 +278,11 @@ query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Lef
         false -> {keep_state, ensure_flush_timer(St)}
     end;
 query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, query_mode := QM} = St) ->
-    case send_query(QM, From, Request, Id) of
+    QueryOpts = #{
+        inflight_name => maps:get(name, St),
+        inflight_window => maps:get(async_inflight_window, St)
+    },
+    case send_query(QM, From, Request, Id, QueryOpts) of
         true ->
             Query = ?QUERY(From, Request),
             {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Query)])}};
@@ -262,8 +290,8 @@ query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, quer
             {keep_state, St}
     end.
 
-send_query(QM, From, Request, Id) ->
-    Result = call_query(QM, Id, ?QUERY(From, Request), 1),
+send_query(QM, From, Request, Id, QueryOpts) ->
+    Result = call_query(QM, Id, ?QUERY(From, Request), QueryOpts),
     reply_caller(Id, ?REPLY(From, Request, Result)).
 
 flush(#{acc := []} = St) ->
@@ -277,7 +305,11 @@ flush(
         query_mode := QM
     } = St
 ) ->
-    Result = call_query(QM, Id, Batch, length(Batch)),
+    QueryOpts = #{
+        inflight_name => maps:get(name, St),
+        inflight_window => maps:get(async_inflight_window, St)
+    },
+    Result = call_query(QM, Id, Batch, QueryOpts),
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
     case batch_reply_caller(Id, Result, Batch) of
         true ->
@@ -332,21 +364,21 @@ handle_query_result(Id, {error, _}, BlockWorker) ->
 handle_query_result(Id, {resource_down, _}, _BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down),
     true;
+handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
+    true;
 handle_query_result(_Id, {async_return, {resource_down, _}}, _BlockWorker) ->
     true;
 handle_query_result(_Id, {async_return, ok}, BlockWorker) ->
     BlockWorker;
 handle_query_result(Id, Result, BlockWorker) ->
-    %% assert
-    true = is_ok_result(Result),
+    assert_ok_result(Result),
     emqx_metrics_worker:inc(?RES_METRICS, Id, success),
     BlockWorker.
 
-call_query(QM, Id, Query, QueryLen) ->
+call_query(QM, Id, Query, QueryOpts) ->
     case emqx_resource_manager:ets_lookup(Id) of
         {ok, _Group, #{callback_mode := CM, mod := Mod, state := ResSt, status := connected}} ->
-            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, QueryLen),
-            apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt);
+            apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt, QueryOpts);
         {ok, _Group, #{status := stopped}} ->
             ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
         {ok, _Group, #{status := S}} when S == connecting; S == disconnected ->
@@ -372,58 +404,119 @@ call_query(QM, Id, Query, QueryLen) ->
     end
 ).
 
-apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt) ->
+apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
     ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request);
-apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt) ->
+apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
-    ReplyFun = fun ?MODULE:reply_after_query/4,
+    Name = maps:get(inflight_name, QueryOpts, undefined),
+    WinSize = maps:get(inflight_window, QueryOpts, undefined),
     ?APPLY_RESOURCE(
-        begin
-            Result = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt),
-            {async_return, Result}
+        case inflight_is_full(Name, WinSize) of
+            true ->
+                ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
+                {async_return, inflight_full};
+            false ->
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+                ReplyFun = fun ?MODULE:reply_after_query/6,
+                Ref = make_message_ref(),
+                Args = [self(), Id, Name, Ref, Query],
+                ok = inflight_append(Name, Ref, Query),
+                Result = Mod:on_query_async(Id, Request, {ReplyFun, Args}, ResSt),
+                {async_return, Result}
         end,
         Request
     );
-apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) ->
+apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
     ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)),
     ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch);
-apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) ->
+apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
-    Requests = [Request || ?QUERY(_From, Request) <- Batch],
-    ReplyFun = fun ?MODULE:batch_reply_after_query/4,
+    Name = maps:get(inflight_name, QueryOpts, undefined),
+    WinSize = maps:get(inflight_window, QueryOpts, undefined),
     ?APPLY_RESOURCE(
-        begin
-            Result = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt),
-            {async_return, Result}
+        case inflight_is_full(Name, WinSize) of
+            true ->
+                ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
+                {async_return, inflight_full};
+            false ->
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)),
+                ReplyFun = fun ?MODULE:batch_reply_after_query/6,
+                Ref = make_message_ref(),
+                Args = {ReplyFun, [self(), Id, Name, Ref, Batch]},
+                Requests = [Request || ?QUERY(_From, Request) <- Batch],
+                ok = inflight_append(Name, Ref, Batch),
+                Result = Mod:on_batch_query_async(Id, Requests, Args, ResSt),
+                {async_return, Result}
         end,
         Batch
     ).
 
-reply_after_query(Pid, Id, ?QUERY(From, Request) = Query, Result) ->
+reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request), Result) ->
     case reply_caller(Id, ?REPLY(From, Request, Result)) of
-        true -> ?MODULE:block(Pid, [Query]);
-        false -> ok
+        true -> ?MODULE:block(Pid);
+        false -> inflight_drop(Name, Ref)
     end.
 
-batch_reply_after_query(Pid, Id, Batch, Result) ->
+batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
     case batch_reply_caller(Id, Result, Batch) of
-        true -> ?MODULE:block(Pid, Batch);
-        false -> ok
+        true -> ?MODULE:block(Pid);
+        false -> inflight_drop(Name, Ref)
     end.
+%%==============================================================================
+%% the inflight queue for async query
+
+inflight_new(Name) ->
+    _ = ets:new(Name, [named_table, ordered_set, public, {write_concurrency, true}]),
+    ok.
+
+inflight_get_first(Name) ->
+    case ets:first(Name) of
+        '$end_of_table' ->
+            empty;
+        Ref ->
+            case ets:lookup(Name, Ref) of
+                [Object] -> Object;
+                [] -> inflight_get_first(Name)
+            end
+    end.
+
+inflight_is_full(undefined, _) ->
+    false;
+inflight_is_full(Name, MaxSize) ->
+    case ets:info(Name, size) of
+        Size when Size >= MaxSize -> true;
+        _ -> false
+    end.
+
+inflight_append(undefined, _Ref, _Query) ->
+    ok;
+inflight_append(Name, Ref, Query) ->
+    ets:insert(Name, {Ref, Query}),
+    ok.
+
+inflight_drop(undefined, _) ->
+    ok;
+inflight_drop(Name, Ref) ->
+    ets:delete(Name, Ref),
+    ok.
 
 %%==============================================================================
 call_mode(sync, _) -> sync;
 call_mode(async, always_sync) -> sync;
 call_mode(async, async_if_possible) -> async.
 
-is_ok_result(ok) ->
+assert_ok_result(ok) ->
     true;
-is_ok_result(R) when is_tuple(R) ->
-    erlang:element(1, R) == ok;
-is_ok_result(_) ->
-    false.
+assert_ok_result({async_return, R}) ->
+    assert_ok_result(R);
+assert_ok_result(R) when is_tuple(R) ->
+    ok = erlang:element(1, R);
+assert_ok_result(R) ->
+    error({not_ok_result, R}).
 
 -spec name(id(), integer()) -> atom().
 name(Id, Index) ->
@@ -447,3 +540,6 @@ cancel_flush_timer(St = #{tref := undefined}) ->
 cancel_flush_timer(St = #{tref := {TRef, _Ref}}) ->
     _ = erlang:cancel_timer(TRef),
     St#{tref => undefined}.
+
+make_message_ref() ->
+    erlang:unique_integer([monotonic, positive]).
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index 3bea71993..6e7bca18a 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -31,7 +31,7 @@
     on_get_status/2
 ]).
 
--export([counter_loop/1, set_callback_mode/1]).
+-export([counter_loop/0, set_callback_mode/1]).
 
 %% callbacks for emqx_resource config schema
 -export([roots/0]).
@@ -84,9 +84,22 @@ on_query(_InstId, get_state, State) ->
     {ok, State};
 on_query(_InstId, get_state_failed, State) ->
     {error, State};
-on_query(_InstId, {inc_counter, N}, #{pid := Pid}) ->
-    Pid ! {inc, N},
+on_query(_InstId, block, #{pid := Pid}) ->
+    Pid ! block,
     ok;
+on_query(_InstId, resume, #{pid := Pid}) ->
+    Pid ! resume,
+    ok;
+on_query(_InstId, {inc_counter, N}, #{pid := Pid}) ->
+    ReqRef = make_ref(),
+    From = {self(), ReqRef},
+    Pid ! {From, {inc, N}},
+    receive
+        {ReqRef, ok} -> ok;
+        {ReqRef, incorrect_status} -> {resource_down, incorrect_status}
+    after 1000 ->
+        {error, timeout}
+    end;
 on_query(_InstId, get_counter, #{pid := Pid}) ->
     ReqRef = make_ref(),
     From = {self(), ReqRef},
@@ -97,9 +110,12 @@ on_query(_InstId, get_counter, #{pid := Pid}) ->
         {error, timeout}
     end.
 
-on_query_async(_InstId, Query, ReplyFun, State) ->
-    Result = on_query(_InstId, Query, State),
-    apply_reply(ReplyFun, Result).
+on_query_async(_InstId, {inc_counter, N}, ReplyFun, #{pid := Pid}) ->
+    Pid ! {inc, N, ReplyFun},
+    ok;
+on_query_async(_InstId, get_counter, ReplyFun, #{pid := Pid}) ->
+    Pid ! {get, ReplyFun},
+    ok.
 
 on_batch_query(InstId, BatchReq, State) ->
     %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed.
@@ -136,15 +152,35 @@ on_get_status(_InstId, #{pid := Pid}) ->
     end.
 
 spawn_counter_process(Name, Register) ->
-    Pid = spawn_link(?MODULE, counter_loop, [#{counter => 0}]),
+    Pid = spawn_link(?MODULE, counter_loop, []),
     true = maybe_register(Name, Pid, Register),
     Pid.
 
-counter_loop(#{counter := Num} = State) ->
+counter_loop() ->
+    counter_loop(#{counter => 0, status => running}).
+
+counter_loop(#{counter := Num, status := Status} = State) ->
     NewState =
         receive
-            {inc, N} ->
-                #{counter => Num + N};
+            block ->
+                ct:pal("counter recv: ~p", [block]),
+                State#{status => blocked};
+            resume ->
+                {messages, Msgs} = erlang:process_info(self(), messages),
+                ct:pal("counter recv: ~p, buffered msgs: ~p", [resume, length(Msgs)]),
+                State#{status => running};
+            {inc, N, ReplyFun} when Status == running ->
+                apply_reply(ReplyFun, ok),
+                State#{counter => Num + N};
+            {{FromPid, ReqRef}, {inc, N}} when Status == running ->
+                FromPid ! {ReqRef, ok},
+                State#{counter => Num + N};
+            {{FromPid, ReqRef}, {inc, _N}} when Status == blocked ->
+                FromPid ! {ReqRef, incorrect_status},
+                State;
+            {get, ReplyFun} ->
+                apply_reply(ReplyFun, Num),
+                State;
             {{FromPid, ReqRef}, get} ->
                 FromPid ! {ReqRef, Num},
                 State
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 5177e792c..5c62e22c2 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -236,7 +236,7 @@ t_batch_query_counter(_) ->
 
     ok = emqx_resource:remove_local(?ID).
 
-t_query_counter_async(_) ->
+t_query_counter_async_query(_) ->
     {ok, _} = emqx_resource:create_local(
         ?ID,
         ?DEFAULT_RESOURCE_GROUP,
@@ -271,24 +271,25 @@ t_query_counter_async(_) ->
     ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C),
     ok = emqx_resource:remove_local(?ID).
 
-t_query_counter_async_2(_) ->
+t_query_counter_async_callback(_) ->
     emqx_connector_demo:set_callback_mode(async_if_possible),
 
     Tab0 = ets:new(?FUNCTION_NAME, [bag, public]),
     Insert = fun(Tab, Result) ->
         ets:insert(Tab, {make_ref(), Result})
     end,
+    ReqOpts = #{async_reply_fun => {Insert, [Tab0]}},
     {ok, _} = emqx_resource:create_local(
         ?ID,
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
         #{name => test_resource, register => true},
-        #{query_mode => async, async_reply_fun => {Insert, [Tab0]}}
+        #{query_mode => async, async_inflight_window => 1000000}
     ),
     ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)),
     ?check_trace(
         ?TRACE_OPTS,
-        inc_counter_in_parallel(1000),
+        inc_counter_in_parallel(1000, ReqOpts),
         fun(Trace) ->
             QueryTrace = ?of_kind(call_query_async, Trace),
             ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
@@ -321,6 +322,117 @@ t_query_counter_async_2(_) ->
     ),
     ok = emqx_resource:remove_local(?ID).
 
+t_query_counter_async_inflight(_) ->
+    emqx_connector_demo:set_callback_mode(async_if_possible),
+
+    Tab0 = ets:new(?FUNCTION_NAME, [bag, public]),
+    Insert0 = fun(Tab, Result) ->
+        ets:insert(Tab, {make_ref(), Result})
+    end,
+    ReqOpts = #{async_reply_fun => {Insert0, [Tab0]}},
+    WindowSize = 15,
+    {ok, _} = emqx_resource:create_local(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource, register => true},
+        #{
+            query_mode => async,
+            async_inflight_window => WindowSize,
+            worker_pool_size => 1,
+            resume_interval => 300
+        }
+    ),
+    ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)),
+
+    %% block the resource
+    ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)),
+
+    %% send async query to make the inflight window full
+    ?check_trace(
+        ?TRACE_OPTS,
+        inc_counter_in_parallel(WindowSize, ReqOpts),
+        fun(Trace) ->
+            QueryTrace = ?of_kind(call_query_async, Trace),
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+        end
+    ),
+
+    %% this will block the resource_worker
+    ok = emqx_resource:query(?ID, {inc_counter, 1}),
+    ?assertMatch(0, ets:info(Tab0, size)),
+    %% sleep to make the resource_worker resume some times
+    timer:sleep(2000),
+
+    %% send query now will fail because the resource is blocked.
+    Insert = fun(Tab, Ref, Result) ->
+        ets:insert(Tab, {Ref, Result})
+    end,
+    ok = emqx_resource:query(?ID, {inc_counter, 1}, #{
+        async_reply_fun => {Insert, [Tab0, tmp_query]}
+    }),
+    ?assertMatch([{_, {error, {resource_error, #{reason := blocked}}}}], ets:take(Tab0, tmp_query)),
+
+    %% all response should be received after the resource is resumed.
+    ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)),
+    timer:sleep(1000),
+    ?assertEqual(WindowSize, ets:info(Tab0, size)),
+
+    %% send async query, this time everything should be ok.
+    Num = 10,
+    ?check_trace(
+        ?TRACE_OPTS,
+        inc_counter_in_parallel(Num, ReqOpts),
+        fun(Trace) ->
+            QueryTrace = ?of_kind(call_query_async, Trace),
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+        end
+    ),
+    timer:sleep(1000),
+    ?assertEqual(WindowSize + Num, ets:info(Tab0, size)),
+
+    %% block the resource
+    ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)),
+    %% again, send async query to make the inflight window full
+    ?check_trace(
+        ?TRACE_OPTS,
+        inc_counter_in_parallel(WindowSize, ReqOpts),
+        fun(Trace) ->
+            QueryTrace = ?of_kind(call_query_async, Trace),
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+        end
+    ),
+
+    %% this will block the resource_worker
+    ok = emqx_resource:query(?ID, {inc_counter, 1}),
+
+    Sent = WindowSize + Num + WindowSize,
+    ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)),
+    timer:sleep(1000),
+    ?assertEqual(Sent, ets:info(Tab0, size)),
+
+    {ok, Counter} = emqx_resource:simple_sync_query(?ID, get_counter),
+    ct:pal("get_counter: ~p, sent: ~p", [Counter, Sent]),
+    ?assert(Sent == Counter),
+
+    {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
+    ct:pal("metrics: ~p", [C]),
+    ?assertMatch(
+        #{matched := M, success := S, exception := E, failed := F, resource_down := RD} when
+            M >= Sent andalso M == S + E + F + RD,
+        C
+    ),
+    ?assert(
+        lists:all(
+            fun
+                ({_, ok}) -> true;
+                (_) -> false
+            end,
+            ets:tab2list(Tab0)
+        )
+    ),
+    ok = emqx_resource:remove_local(?ID).
+
 t_healthy_timeout(_) ->
     {ok, _} = emqx_resource:create_local(
         ?ID,
@@ -550,10 +662,13 @@ t_auto_retry(_) ->
 %% Helpers
 %%------------------------------------------------------------------------------
 inc_counter_in_parallel(N) ->
+    inc_counter_in_parallel(N, #{}).
+
+inc_counter_in_parallel(N, Opts) ->
     Parent = self(),
     Pids = [
         erlang:spawn(fun() ->
-            emqx_resource:query(?ID, {inc_counter, 1}),
+            emqx_resource:query(?ID, {inc_counter, 1}, Opts),
             Parent ! {complete, self()}
         end)
      || _ <- lists:seq(1, N)
@@ -567,6 +682,17 @@ inc_counter_in_parallel(N) ->
      || Pid <- Pids
     ].
 
+% verify_inflight_full(WindowSize) ->
+%     ?check_trace(
+%         ?TRACE_OPTS,
+%         emqx_resource:query(?ID, {inc_counter, 1}),
+%         fun(Return, Trace) ->
+%             QueryTrace = ?of_kind(inflight_full, Trace),
+%             ?assertMatch([#{wind_size := WindowSize} | _], QueryTrace),
+%             ?assertMatch(ok, Return)
+%         end
+%     ).
+
 bin_config() ->
     <<"\"name\": \"test_resource\"">>.
 

From 0090a3ee9393f00ecfd6285dab79979a206eacda Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 10 Aug 2022 11:47:14 +0800
Subject: [PATCH 044/232] fix(influxdb): fix illegal `wirte_syntax` example

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 83c5a4127..53d02fa6d 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -51,7 +51,7 @@ values(Protocol, get) ->
 values(Protocol, post) ->
     case Protocol of
         "influxdb_api_v2" ->
-            SupportUint = <<"uint_value=${payload.uint_key}u">>;
+            SupportUint = <<"uint_value=${payload.uint_key}u,">>;
         _ ->
             SupportUint = <<>>
     end,

From 223b84017edd0d557498a184678c15b3590a6340 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 11:35:04 +0800
Subject: [PATCH 045/232] fix(influxdb): api schema `write_syntax` using raw
 type `string()`

---
 apps/emqx_dashboard/src/emqx_dashboard_swagger.erl    |  7 +++++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 11 ++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 59b320368..68514fdb0 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -656,6 +656,13 @@ typename_to_spec("file()", _Mod) ->
     #{type => string, example => <<"/path/to/file">>};
 typename_to_spec("ip_port()", _Mod) ->
     #{type => string, example => <<"127.0.0.1:80">>};
+typename_to_spec("write_syntax()", _Mod) ->
+    #{
+        type => string,
+        example =>
+            <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
+                "${clientid}_int_value=${payload.int_key}i,", "bool=${payload.bool}">>
+    };
 typename_to_spec("ip_ports()", _Mod) ->
     #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
 typename_to_spec("url()", _Mod) ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 53d02fa6d..7de640040 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -21,6 +21,11 @@
     desc/1
 ]).
 
+-type write_syntax() :: list().
+-reflect_type([write_syntax/0]).
+-typerefl_from_string({write_syntax/0, ?MODULE, to_influx_lines}).
+-export([to_influx_lines/1]).
+
 %% -------------------------------------------------------------------------------------------------
 %% api
 
@@ -148,19 +153,19 @@ desc(_) ->
     undefined.
 
 write_syntax(type) ->
-    list();
+    ?MODULE:write_syntax();
 write_syntax(required) ->
     true;
 write_syntax(validator) ->
     [?NOT_EMPTY("the value of the field 'write_syntax' cannot be empty")];
 write_syntax(converter) ->
-    fun converter_influx_lines/1;
+    fun to_influx_lines/1;
 write_syntax(desc) ->
     ?DESC("write_syntax");
 write_syntax(_) ->
     undefined.
 
-converter_influx_lines(RawLines) ->
+to_influx_lines(RawLines) ->
     Lines = string:tokens(str(RawLines), "\n"),
     lists:reverse(lists:foldl(fun converter_influx_line/2, [], Lines)).
 

From 7581082fcbd86b6090cc93a2b9345b6bcb52cd30 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 11 Aug 2022 13:22:16 +0800
Subject: [PATCH 046/232] fix(bridge): replace prepare_statement by
 sql_template

---
 .../src/emqx_connector_mysql.erl               | 16 ++++++++++++++--
 .../i18n/emqx_ee_bridge_mysql.conf             |  2 +-
 .../src/emqx_ee_bridge_mysql.erl               | 18 ++++--------------
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index b379e511c..cae07433a 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -111,7 +111,7 @@ on_start(
         {pool_size, PoolSize}
     ],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
-    Prepares = parse_prepare_sql(maps:get(prepare_statement, Config, #{})),
+    Prepares = parse_prepare_sql(Config),
     State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
         ok -> {ok, init_prepare(State)};
@@ -303,7 +303,19 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) ->
 unprepare_sql_to_conn(Conn, PrepareSqlKey) ->
     mysql:unprepare(Conn, PrepareSqlKey).
 
-parse_prepare_sql(SQL) ->
+parse_prepare_sql(Config) ->
+    SQL =
+        case maps:get(prepare_statement, Config, undefined) of
+            undefined ->
+                case emqx_map_lib:deep_get([egress, sql_template], Config, undefined) of
+                    undefined ->
+                        #{};
+                    Template ->
+                        #{send_message => Template}
+                end;
+            Any ->
+                Any
+        end,
     parse_prepare_sql(maps:to_list(SQL), #{}, #{}).
 
 parse_prepare_sql([{Key, H} | T], SQL, Tokens) ->
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
index 9e3796487..48fbd1007 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
@@ -1,5 +1,5 @@
 emqx_ee_bridge_mysql {
-    sql {
+    sql_template {
         desc {
             en: """SQL Template"""
             zh: """SQL 模板"""
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index bf38da2f8..ebcbc0e64 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -44,14 +44,14 @@ values(post) ->
     #{
         type => mysql,
         name => <<"mysql">>,
+        sql_template => ?DEFAULT_SQL,
         connector => #{
             server => <<"127.0.0.1:3306">>,
             database => <<"test">>,
             pool_size => 8,
             username => <<"root">>,
             password => <<"public">>,
-            auto_reconnect => true,
-            prepare_statement => #{send_message => ?DEFAULT_SQL}
+            auto_reconnect => true
         },
         enable => true,
         direction => egress
@@ -69,6 +69,7 @@ fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
+        {sql_template, mk(binary(), #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL})},
         {connector,
             mk(
                 ref(?MODULE, connector),
@@ -85,8 +86,7 @@ fields("put") ->
 fields("get") ->
     emqx_bridge_schema:metrics_status_fields() ++ fields("post");
 fields(connector) ->
-    (emqx_connector_mysql:fields(config) --
-        emqx_connector_schema_lib:prepare_statement_fields()) ++ prepare_statement_fields().
+    emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields().
 
 desc("config") ->
     ?DESC("desc_config");
@@ -104,13 +104,3 @@ type_field() ->
 
 name_field() ->
     {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
-
-prepare_statement_fields() ->
-    [
-        {prepare_statement,
-            mk(map(), #{
-                desc => ?DESC(emqx_connector_schema_lib, prepare_statement),
-                default => #{<<"send_message">> => ?DEFAULT_SQL},
-                example => #{<<"send_message">> => ?DEFAULT_SQL}
-            })}
-    ].

From dc804993999de0b46175d00e59b3521c016f45e5 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 11 Aug 2022 15:58:48 +0800
Subject: [PATCH 047/232] fix(bridge): add sql_template field format

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index ebcbc0e64..5d143bf85 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -69,7 +69,11 @@ fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
-        {sql_template, mk(binary(), #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL})},
+        {sql_template,
+            mk(
+                binary(),
+                #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>}
+            )},
         {connector,
             mk(
                 ref(?MODULE, connector),

From 22a4ca311c9e9320260d90d9d2eb2ecc2c1d6cd5 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 15:09:05 +0800
Subject: [PATCH 048/232] feat(resource): resource batch/async/queue config
 schema

---
 .../i18n/emqx_resource_schema_i18n.conf       | 119 ++++++++++++++++++
 apps/emqx_resource/include/emqx_resource.hrl  |  12 ++
 .../src/emqx_resource_worker.erl              |  17 +--
 .../src/schema/emqx_resource_schema.erl       |  99 +++++++++++++++
 .../test/emqx_resource_SUITE.erl              |   2 +-
 5 files changed, 235 insertions(+), 14 deletions(-)
 create mode 100644 apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
 create mode 100644 apps/emqx_resource/src/schema/emqx_resource_schema.erl

diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
new file mode 100644
index 000000000..a3fb6c402
--- /dev/null
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -0,0 +1,119 @@
+emqx_resource_schema {
+  batch {
+    desc {
+      en: """
+Configuration of batch query.</br>
+Batch requests are made immediately when the number of requests reaches the `batch_size`, or also immediately when the number of requests is less than the batch request size but the maximum batch_time has been reached.
+"""
+      zh: """
+批量请求配置。</br>
+请求数达到批量请求大小时立刻进行批量请求，或当请求数不足批量请求数大小，但已经达到最大批量等待时间时也立即进行批量请求。
+"""
+    }
+    label {
+      en: """batch"""
+      zh: """批量请求"""
+    }
+  }
+
+  queue {
+    desc {
+      en: """Configuration of queue."""
+      zh: """请求队列配置"""
+    }
+    label {
+      en: """queue"""
+      zh: """请求队列"""
+    }
+  }
+
+  query_mode {
+    desc {
+      en: """Query mode. Optional 'sync/async', default 'sync'."""
+      zh: """请求模式。可选 '同步/异步'，默认为'同步'模式。"""
+    }
+    label {
+      en: """query_mode"""
+      zh: """query_mode"""
+    }
+  }
+
+  enable_batch {
+    desc {
+      en: """Batch mode enabled."""
+      zh: """启用批量模式。"""
+    }
+    label {
+      en: """enable_batch"""
+      zh: """启用批量模式"""
+    }
+  }
+
+  enable_queue {
+    desc {
+      en: """Queue mode enabled."""
+      zh: """启用队列模式。"""
+    }
+    label {
+      en: """enable_queue"""
+      zh: """启用队列模式"""
+    }
+  }
+
+  resume_interval {
+    desc {
+      en: """Resume time interval when resource down."""
+      zh: """资源不可用时的重试时间"""
+    }
+    label {
+      en: """resume_interval"""
+      zh: """恢复时间"""
+    }
+  }
+
+  async_inflight_window {
+    desc {
+      en: """Async queyr inflight window."""
+      zh: """异步请求飞行队列窗口大小"""
+    }
+    label {
+      en: """async_inflight_window"""
+      zh: """异步请求飞行队列窗口"""
+    }
+  }
+
+  batch_size {
+    desc {
+      en: """Maximum batch count."""
+      zh: """批量请求大小"""
+    }
+    label {
+      en: """batch_size"""
+      zh: """批量请求大小"""
+    }
+  }
+
+  batch_time {
+    desc {
+      en: """Maximum batch waiting interval."""
+      zh: """最大批量请求等待时间。"""
+    }
+    label {
+      en: """batch_time"""
+      zh: """批量等待间隔"""
+    }
+  }
+
+  queue_max_bytes {
+    desc {
+      en: """Maximum queue storage size in bytes."""
+      zh: """消息队列的最大长度，以字节计。"""
+    }
+    label {
+      en: """queue_max_bytes"""
+      zh: """队列最大长度"""
+    }
+  }
+
+
+}
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 5c561a8d3..5327e3aae 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -60,5 +60,17 @@
     | {error, term()}
     | {resource_down, term()}.
 
+%% count
+-define(DEFAULT_BATCH_SIZE, 100).
+%% milliseconds
+-define(DEFAULT_BATCH_TIME, 10).
+
+%% bytes
+-define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
+
+-define(DEFAULT_INFLIGHT, 100).
+
+-define(RESUME_INTERVAL, 15000).
+
 -define(TEST_ID_PREFIX, "_test_:").
 -define(RES_METRICS, resource_metrics).
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 716842bf9..e940dcb69 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -52,13 +52,6 @@
 
 -export([reply_after_query/6, batch_reply_after_query/6]).
 
--define(RESUME_INTERVAL, 15000).
-
-%% count
--define(DEFAULT_BATCH_SIZE, 100).
-%% milliseconds
--define(DEFAULT_BATCH_TIME, 10).
-
 -define(Q_ITEM(REQUEST), {q_item, REQUEST}).
 
 -define(QUERY(FROM, REQUEST), {query, FROM, REQUEST}).
@@ -69,8 +62,6 @@
     {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}}
 ).
 -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
--define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
--define(DEFAULT_INFLIGHT, 100).
 
 -type id() :: binary().
 -type query() :: {query, from(), request()}.
@@ -122,7 +113,7 @@ init({Id, Index, Opts}) ->
     Name = name(Id, Index),
     BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE),
     Queue =
-        case maps:get(queue_enabled, Opts, false) of
+        case maps:get(enable_queue, Opts, false) of
             true ->
                 replayq:open(#{
                     dir => disk_queue_dir(Id, Index),
@@ -144,7 +135,7 @@ init({Id, Index, Opts}) ->
         %%  if the resource worker is overloaded
         query_mode => maps:get(query_mode, Opts, sync),
         async_inflight_window => maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT),
-        batch_enabled => maps:get(batch_enabled, Opts, false),
+        enable_batch => maps:get(enable_batch, Opts, false),
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
         queue => Queue,
@@ -270,14 +261,14 @@ drop_head(Q) ->
     ok = replayq:ack(Q1, AckRef),
     Q1.
 
-query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Left} = St0) ->
+query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left} = St0) ->
     Acc1 = [?QUERY(From, Request) | Acc],
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
         true -> flush(St);
         false -> {keep_state, ensure_flush_timer(St)}
     end;
-query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, query_mode := QM} = St) ->
+query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id, query_mode := QM} = St) ->
     QueryOpts = #{
         inflight_name => maps:get(name, St),
         inflight_window => maps:get(async_inflight_window, St)
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
new file mode 100644
index 000000000..933cd0189
--- /dev/null
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -0,0 +1,99 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_resource_schema).
+
+-include("emqx_resource.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-export([namespace/0, roots/0, fields/1]).
+
+%% -------------------------------------------------------------------------------------------------
+%% Hocon Schema Definitions
+
+namespace() -> "resource_schema".
+
+roots() -> [].
+
+fields('batch&async&queue') ->
+    [
+        {query_mode, fun query_mode/1},
+        {resume_interval, fun resume_interval/1},
+        {async_inflight_window, fun async_inflight_window/1},
+        {batch, mk(ref(?MODULE, batch), #{desc => ?DESC("batch")})},
+        {queue, mk(ref(?MODULE, queue), #{desc => ?DESC("queue")})}
+    ];
+fields(batch) ->
+    [
+        {enable_batch, fun enable_batch/1},
+        {batch_size, fun batch_size/1},
+        {batch_time, fun batch_time/1}
+    ];
+fields(queue) ->
+    [
+        {enable_queue, fun enable_queue/1},
+        {max_queue_bytes, fun queue_max_bytes/1}
+    ].
+
+query_mode(type) -> enum([sync, async]);
+query_mode(desc) -> ?DESC("query_mode");
+query_mode(default) -> sync;
+query_mode(required) -> false;
+query_mode(_) -> undefined.
+
+enable_batch(type) -> boolean();
+enable_batch(required) -> false;
+enable_batch(default) -> false;
+enable_batch(desc) -> ?DESC("enable_batch");
+enable_batch(_) -> undefined.
+
+enable_queue(type) -> boolean();
+enable_queue(required) -> false;
+enable_queue(default) -> false;
+enable_queue(desc) -> ?DESC("enable_queue");
+enable_queue(_) -> undefined.
+
+resume_interval(type) -> emqx_schema:duration_ms();
+resume_interval(desc) -> ?DESC("resume_interval");
+resume_interval(default) -> ?RESUME_INTERVAL;
+resume_interval(required) -> false;
+resume_interval(_) -> undefined.
+
+async_inflight_window(type) -> pos_integer();
+async_inflight_window(desc) -> ?DESC("async_inflight_window");
+async_inflight_window(default) -> ?DEFAULT_INFLIGHT;
+async_inflight_window(required) -> false;
+async_inflight_window(_) -> undefined.
+
+batch_size(type) -> pos_integer();
+batch_size(desc) -> ?DESC("batch_size");
+batch_size(default) -> ?DEFAULT_BATCH_SIZE;
+batch_size(required) -> false;
+batch_size(_) -> undefined.
+
+batch_time(type) -> emqx_schema:duration_ms();
+batch_time(desc) -> ?DESC("batch_time");
+batch_time(default) -> ?DEFAULT_BATCH_TIME;
+batch_time(required) -> false;
+batch_time(_) -> undefined.
+
+queue_max_bytes(type) -> emqx_schema:bytesize();
+queue_max_bytes(desc) -> ?DESC("queue_max_bytes");
+queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE;
+queue_max_bytes(required) -> false;
+queue_max_bytes(_) -> undefined.
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 5c62e22c2..ddd671b75 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -211,7 +211,7 @@ t_batch_query_counter(_) ->
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
         #{name => test_resource, register => true},
-        #{batch_enabled => true}
+        #{enable_batch => true}
     ),
 
     ?check_trace(

From 2843d33f7a33fa46bb12ae09b788944b99870403 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 15:10:25 +0800
Subject: [PATCH 049/232] chore: refine md form format

---
 deploy/charts/emqx/README.md | 108 +++++++++++++++++------------------
 1 file changed, 54 insertions(+), 54 deletions(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 496d52061..acbbcf95e 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -1,5 +1,5 @@
 # Introduction
-This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. 
+This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager.
 
 # Prerequisites
 + Kubernetes 1.6+
@@ -8,7 +8,7 @@ This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm
 # Installing the Chart
 To install the chart with the release name `my-emqx`:
 
-+   From github 
++   From github
     ```
     $ git clone https://github.com/emqx/emqx.git
     $ cd emqx/deploy/charts/emqx
@@ -31,58 +31,58 @@ $ helm del  my-emqx
 # Configuration
 The following table lists the configurable parameters of the emqx chart and their default values.
 
-| Parameter  | Description | Default Value |
-| ---        |  ---        | ---           |
-| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. |3|
-| `image.repository` | EMQX Image name |emqx/emqx|
-| `image.pullPolicy`  | The image pull policy  |IfNotPresent|
-| `image.pullSecrets `  | The image pull secrets  |`[]` (does not add image pull secrets to deployed pods)|
-| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil |
-| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false |
-`podAnnotations ` | Annotations for pod | `{}`
-`podManagementPolicy`| To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel`
-| `persistence.enabled` | Enable EMQX persistence using PVC |false|
-| `persistence.storageClass` | Storage class of backing PVC |`nil` (uses alpha storage class annotation)|
-| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template |""|
-| `persistence.accessMode` | PVC Access Mode for EMQX volume |ReadWriteOnce|
-| `persistence.size` | PVC Storage Request for EMQX volume |20Mi|
-| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`|
-| `resources` | CPU/Memory resource requests/limits |{}|
-| `nodeSelector` | Node labels for pod assignment |`{}`|
-| `tolerations` | Toleration labels for pod assignment |`[]`|
-| `affinity` | Map of node/pod affinities |`{}`|
-| `service.type`  | Kubernetes Service type. |ClusterIP|
-| `service.mqtt`  | Port for MQTT. |1883|
-| `service.mqttssl` | Port for MQTT(SSL). |8883|
-| `service.mgmt`  | Port for mgmt API. |8081|
-| `service.ws`  | Port for WebSocket/HTTP. |8083|
-| `service.wss`  | Port for WSS/HTTPS. |8084|
-| `service.dashboard`  | Port for dashboard. |18083|
-| `service.nodePorts.mqtt`  | Kubernetes node port for MQTT. |nil|
-| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). |nil|
-| `service.nodePorts.mgmt`  | Kubernetes node port for mgmt API. |nil|
-| `service.nodePorts.ws`  | Kubernetes node port for WebSocket/HTTP. |nil|
-| `service.nodePorts.wss`  | Kubernetes node port for WSS/HTTPS. |nil|
-| `service.nodePorts.dashboard`  | Kubernetes node port for dashboard. |nil|
-| `service.loadBalancerIP`  | loadBalancerIP for Service |	nil |
-| `service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	[] |
-| `service.externalIPs` |	ExternalIPs for the service |	[] |
-| `service.annotations` |	Service annotations |	{}(evaluated as a template)|
-| `ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false |
-| `ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard |	 |
-| `ingress.dashboard.path` | Ingress path for EMQX Dashboard |	/ |
-| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard |	`ImplementationSpecific`
-| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API |	dashboard.emqx.local |
-| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API |	[] |
-| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API |	{} |
-| `ingress.mgmt.enabled` |	Enable ingress for EMQX Mgmt API |	false |
-| `ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Mgmt API |	 |
-| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API |	/ |
-| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API |	api.emqx.local |
-| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API |	[] |
-| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API |	{} |
-| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false |
-| `metrics.type` | Now we only supported "prometheus" | "prometheus" |
+| Parameter                            | Description                                                                                                                                                  | Default Value                                           |
+|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|
+| `replicaCount`                       | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split.                  | 3                                                       |
+| `image.repository`                   | EMQX Image name                                                                                                                                              | emqx/emqx                                               |
+| `image.pullPolicy`                   | The image pull policy                                                                                                                                        | IfNotPresent                                            |
+| `image.pullSecrets `                 | The image pull secrets                                                                                                                                       | `[]` (does not add image pull secrets to deployed pods) |
+| `envFromSecret`                      | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment                                          | nil                                                     |
+| `recreatePods`                       | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration.                                            | false                                                   |
+| `podAnnotations `                    | Annotations for pod                                                                                                                                          | `{}`                                                    |
+| `podManagementPolicy`                | To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock                                                                | `Parallel`                                              |
+| `persistence.enabled`                | Enable EMQX persistence using PVC                                                                                                                            | false                                                   |
+| `persistence.storageClass`           | Storage class of backing PVC                                                                                                                                 | `nil` (uses alpha storage class annotation)             |
+| `persistence.existingClaim`          | EMQX data Persistent Volume existing claim name, evaluated as a template                                                                                     | ""                                                      |
+| `persistence.accessMode`             | PVC Access Mode for EMQX volume                                                                                                                              | ReadWriteOnce                                           |
+| `persistence.size`                   | PVC Storage Request for EMQX volume                                                                                                                          | 20Mi                                                    |
+| `initContainers`                     | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts.                                                     | `{}`                                                    |
+| `resources`                          | CPU/Memory resource requests/limits                                                                                                                          | {}                                                      |
+| `nodeSelector`                       | Node labels for pod assignment                                                                                                                               | `{}`                                                    |
+| `tolerations`                        | Toleration labels for pod assignment                                                                                                                         | `[]`                                                    |
+| `affinity`                           | Map of node/pod affinities                                                                                                                                   | `{}`                                                    |
+| `service.type`                       | Kubernetes Service type.                                                                                                                                     | ClusterIP                                               |
+| `service.mqtt`                       | Port for MQTT.                                                                                                                                               | 1883                                                    |
+| `service.mqttssl`                    | Port for MQTT(SSL).                                                                                                                                          | 8883                                                    |
+| `service.mgmt`                       | Port for mgmt API.                                                                                                                                           | 8081                                                    |
+| `service.ws`                         | Port for WebSocket/HTTP.                                                                                                                                     | 8083                                                    |
+| `service.wss`                        | Port for WSS/HTTPS.                                                                                                                                          | 8084                                                    |
+| `service.dashboard`                  | Port for dashboard.                                                                                                                                          | 18083                                                   |
+| `service.nodePorts.mqtt`             | Kubernetes node port for MQTT.                                                                                                                               | nil                                                     |
+| `service.nodePorts.mqttssl`          | Kubernetes node port for MQTT(SSL).                                                                                                                          | nil                                                     |
+| `service.nodePorts.mgmt`             | Kubernetes node port for mgmt API.                                                                                                                           | nil                                                     |
+| `service.nodePorts.ws`               | Kubernetes node port for WebSocket/HTTP.                                                                                                                     | nil                                                     |
+| `service.nodePorts.wss`              | Kubernetes node port for WSS/HTTPS.                                                                                                                          | nil                                                     |
+| `service.nodePorts.dashboard`        | Kubernetes node port for dashboard.                                                                                                                          | nil                                                     |
+| `service.loadBalancerIP`             | loadBalancerIP for Service                                                                                                                                   | nil                                                     |
+| `service.loadBalancerSourceRanges`   | Address(es) that are allowed when service is LoadBalancer                                                                                                    | []                                                      |
+| `service.externalIPs`                | ExternalIPs for the service                                                                                                                                  | []                                                      |
+| `service.annotations`                | Service annotations                                                                                                                                          | {}(evaluated as a template)                             |
+| `ingress.dashboard.enabled`          | Enable ingress for EMQX Dashboard                                                                                                                            | false                                                   |
+| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard                                                                                                                     |                                                         |
+| `ingress.dashboard.path`             | Ingress path for EMQX Dashboard                                                                                                                              | /                                                       |
+| `ingress.dashboard.pathType`         | Ingress pathType for EMQX Dashboard                                                                                                                          | `ImplementationSpecific`                                |
+| `ingress.dashboard.hosts`            | Ingress hosts for EMQX Mgmt API                                                                                                                              | dashboard.emqx.local                                    |
+| `ingress.dashboard.tls`              | Ingress tls for EMQX Mgmt API                                                                                                                                | []                                                      |
+| `ingress.dashboard.annotations`      | Ingress annotations for EMQX Mgmt API                                                                                                                        | {}                                                      |
+| `ingress.mgmt.enabled`               | Enable ingress for EMQX Mgmt API                                                                                                                             | false                                                   |
+| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API                                                                                                                      |                                                         |
+| `ingress.mgmt.path`                  | Ingress path for EMQX Mgmt API                                                                                                                               | /                                                       |
+| `ingress.mgmt.hosts`                 | Ingress hosts for EMQX Mgmt API                                                                                                                              | api.emqx.local                                          |
+| `ingress.mgmt.tls`                   | Ingress tls for EMQX Mgmt API                                                                                                                                | []                                                      |
+| `ingress.mgmt.annotations`           | Ingress annotations for EMQX Mgmt API                                                                                                                        | {}                                                      |
+| `metrics.enable`                     | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false                                                   |
+| `metrics.type`                       | Now we only supported "prometheus"                                                                                                                           | "prometheus"                                            |
 
 ## EMQX specific settings
 The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their default values.

From 0f6c3717602e20f77635831ea25eaf90b3d2ff7e Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 18:12:41 +0800
Subject: [PATCH 050/232] feat(influxdb): influxdb connector add
 `on_batch_query/3` callback

---
 apps/emqx_resource/src/emqx_resource.erl      |   1 +
 .../src/emqx_ee_bridge_influxdb.erl           |   9 +-
 .../src/emqx_ee_connector_influxdb.erl        | 106 ++++++++++++++----
 3 files changed, 91 insertions(+), 25 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 0d2289696..6601b9eea 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -126,6 +126,7 @@
 %% when calling emqx_resource:query/3
 -callback on_query(resource_id(), Request :: term(), resource_state()) -> query_result().
 
+%% when calling emqx_resource:on_batch_query/3
 -callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result().
 
 %% when calling emqx_resource:health_check/2
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 7de640040..4edeb786a 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -70,7 +70,8 @@ values(Protocol, post) ->
         write_syntax =>
             <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
                 "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary,
-                "bool=${payload.bool}">>
+                "bool=${payload.bool}">>,
+        batch => #{enable_batch => false, batch_size => 5, batch_time => <<"1m">>}
     };
 values(Protocol, put) ->
     values(Protocol, post).
@@ -109,7 +110,9 @@ fields("get_api_v2") ->
 fields(Name) when
     Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2
 ->
-    fields(basic) ++ connector_field(Name).
+    fields(basic) ++
+        emqx_resource_schema:fields('batch&async&queue') ++
+        connector_field(Name).
 
 method_fileds(post, ConnectorType) ->
     fields(basic) ++ connector_field(ConnectorType) ++ type_name_field(ConnectorType);
@@ -162,6 +165,8 @@ write_syntax(converter) ->
     fun to_influx_lines/1;
 write_syntax(desc) ->
     ?DESC("write_syntax");
+write_syntax(format) ->
+    <<"sql">>;
 write_syntax(_) ->
     undefined.
 
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 09b3d7350..41f7059ec 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -17,6 +17,7 @@
     on_start/2,
     on_stop/2,
     on_query/3,
+    on_batch_query/3,
     on_get_status/2
 ]).
 
@@ -37,8 +38,29 @@ on_start(InstId, Config) ->
 on_stop(_InstId, #{client := Client}) ->
     influxdb:stop_client(Client).
 
-on_query(InstId, {send_message, Data}, State) ->
-    do_query(InstId, {send_message, Data}, State).
+on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, client := Client}) ->
+    case data_to_points(Data, SyntaxLines) of
+        {ok, Points} ->
+            do_query(InstId, Client, Points);
+        {error, ErrorPoints} = Err ->
+            log_error_points(InstId, ErrorPoints),
+            Err
+    end.
+
+%% Once a Batched Data trans to points failed.
+%% This batch query failed
+on_batch_query(InstId, BatchData, State = #{write_syntax := SyntaxLines, client := Client}) ->
+    case on_get_status(InstId, State) of
+        connected ->
+            case parse_batch_data(InstId, BatchData, SyntaxLines) of
+                {ok, Points} ->
+                    do_query(InstId, Client, Points);
+                {error, Reason} ->
+                    {error, Reason}
+            end;
+        disconnected ->
+            {resource_down, disconnected}
+    end.
 
 on_get_status(_InstId, #{client := Client}) ->
     case influxdb:is_alive(Client) of
@@ -79,7 +101,7 @@ fields("api_v2_put") ->
 fields(basic) ->
     [
         {host,
-            mk(binary(), #{required => true, default => <<"120.0.0.1">>, desc => ?DESC("host")})},
+            mk(binary(), #{required => true, default => <<"127.0.0.1">>, desc => ?DESC("host")})},
         {port, mk(pos_integer(), #{required => true, default => 8086, desc => ?DESC("port")})},
         {precision,
             mk(enum([ns, us, ms, s, m, h]), #{
@@ -310,18 +332,7 @@ ssl_config(SSL = #{enable := true}) ->
 %% -------------------------------------------------------------------------------------------------
 %% Query
 
-do_query(InstId, {send_message, Data}, State = #{client := Client}) ->
-    {Points, Errs} = data_to_points(Data, State),
-    lists:foreach(
-        fun({error, Reason}) ->
-            ?SLOG(error, #{
-                msg => "influxdb trans point failed",
-                connector => InstId,
-                reason => Reason
-            })
-        end,
-        Errs
-    ),
+do_query(InstId, Client, Points) ->
     case influxdb:write(Client, Points) of
         ok ->
             ?SLOG(debug, #{
@@ -376,11 +387,45 @@ to_maps_config(K, V, Res) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% Tags & Fields Data Trans
-data_to_points(Data, #{write_syntax := Lines}) ->
-    lines_to_points(Data, Lines, [], []).
+parse_batch_data(InstId, BatchData, SyntaxLines) ->
+    {Points, Errors} = lists:foldl(
+        fun({send_message, Data}, {AccIn, ErrAccIn}) ->
+            case data_to_points(Data, SyntaxLines) of
+                {ok, Points} ->
+                    {[Points | AccIn], ErrAccIn};
+                {error, ErrorPoints} ->
+                    log_error_points(InstId, ErrorPoints),
+                    {AccIn, ErrAccIn + 1}
+            end
+        end,
+        {[], 0},
+        BatchData
+    ),
+    case Errors of
+        0 ->
+            {ok, Points};
+        _ ->
+            ?SLOG(error, #{
+                msg => io_lib:format("InfluxDB trans point failed, count: ~p", [Errors]),
+                connector => InstId,
+                reason => points_trans_failed
+            }),
+            {error, points_trans_failed}
+    end.
 
-lines_to_points(_, [], Points, Errs) ->
-    {Points, Errs};
+data_to_points(Data, SyntaxLines) ->
+    lines_to_points(Data, SyntaxLines, [], []).
+
+%% When converting multiple rows data into InfluxDB Line Protocol, they are considered to be strongly correlated.
+%% And once a row fails to convert, all of them are considered to have failed.
+lines_to_points(_, [], Points, ErrorPoints) ->
+    case ErrorPoints of
+        [] ->
+            {ok, Points};
+        _ ->
+            %% ignore trans succeeded points
+            {error, ErrorPoints}
+    end;
 lines_to_points(
     Data,
     [
@@ -392,8 +437,8 @@ lines_to_points(
         }
         | Rest
     ],
-    ResAcc,
-    ErrAcc
+    ResultPointsAcc,
+    ErrorPointsAcc
 ) ->
     TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
     case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of
@@ -406,9 +451,11 @@ lines_to_points(
                 tags => EncodeTags,
                 fields => EncodeFields
             },
-            lines_to_points(Data, Rest, [Point | ResAcc], ErrAcc);
+            lines_to_points(Data, Rest, [Point | ResultPointsAcc], ErrorPointsAcc);
         BadTimestamp ->
-            lines_to_points(Data, Rest, ResAcc, [{error, {bad_timestamp, BadTimestamp}} | ErrAcc])
+            lines_to_points(Data, Rest, ResultPointsAcc, [
+                {error, {bad_timestamp, BadTimestamp}} | ErrorPointsAcc
+            ])
     end.
 
 maps_config_to_data(K, V, {Data, Res}) ->
@@ -461,3 +508,16 @@ data_filter(Bool) when is_boolean(Bool) -> Bool;
 data_filter(Data) -> bin(Data).
 
 bin(Data) -> emqx_plugin_libs_rule:bin(Data).
+
+%% helper funcs
+log_error_points(InstId, Errs) ->
+    lists:foreach(
+        fun({error, Reason}) ->
+            ?SLOG(error, #{
+                msg => "influxdb trans point failed",
+                connector => InstId,
+                reason => Reason
+            })
+        end,
+        Errs
+    ).

From 2872f0b6685061fa4242d829b6f33cdf88e5f76c Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 11 Aug 2022 19:11:44 +0800
Subject: [PATCH 051/232] fix(bridges): support create resources with options

---
 apps/emqx_bridge/src/emqx_bridge.erl          |  9 +++--
 apps/emqx_bridge/src/emqx_bridge_resource.erl | 11 +++--
 .../i18n/emqx_resource_schema_i18n.conf       | 28 -------------
 apps/emqx_resource/include/emqx_resource.hrl  | 12 +++++-
 apps/emqx_resource/src/emqx_resource.erl      | 40 ++++++++++++++-----
 .../src/emqx_resource_manager.erl             |  8 ++--
 .../src/proto/emqx_resource_proto_v1.erl      |  4 +-
 .../src/schema/emqx_resource_schema.erl       | 12 +-----
 .../src/emqx_ee_bridge_influxdb.erl           |  2 +-
 9 files changed, 61 insertions(+), 65 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index ba6c64dbc..c794a25a5 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -171,9 +171,9 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnv) ->
         diff_confs(NewConf, OldConf),
     %% The config update will be failed if any task in `perform_bridge_changes` failed.
     Result = perform_bridge_changes([
-        {fun emqx_bridge_resource:remove/3, Removed},
-        {fun emqx_bridge_resource:create/3, Added},
-        {fun emqx_bridge_resource:update/3, Updated}
+        {fun emqx_bridge_resource:remove/4, Removed},
+        {fun emqx_bridge_resource:create/4, Added},
+        {fun emqx_bridge_resource:update/4, Updated}
     ]),
     ok = unload_hook(),
     ok = load_hook(NewConf),
@@ -261,7 +261,8 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) ->
             ({_Type, _Name}, _Conf, {error, Reason}) ->
                 {error, Reason};
             ({Type, Name}, Conf, _) ->
-                case Action(Type, Name, Conf) of
+                ResOpts = emqx_resource:fetch_creation_opts(Conf),
+                case Action(Type, Name, Conf, ResOpts) of
                     {error, Reason} -> {error, Reason};
                     Return -> Return
                 end
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index 35ace560c..ac1ec6ba3 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -158,11 +158,14 @@ recreate(Type, Name) ->
     recreate(Type, Name, emqx:get_config([bridges, Type, Name])).
 
 recreate(Type, Name, Conf) ->
+    recreate(Type, Name, Conf, #{}).
+
+recreate(Type, Name, Conf, Opts) ->
     emqx_resource:recreate_local(
         resource_id(Type, Name),
         bridge_to_resource_type(Type),
         parse_confs(Type, Name, Conf),
-        #{auto_retry_interval => 60000}
+        Opts#{auto_retry_interval => 60000}
     ).
 
 create_dry_run(Type, Conf) ->
@@ -186,13 +189,13 @@ create_dry_run(Type, Conf) ->
 
 remove(BridgeId) ->
     {BridgeType, BridgeName} = parse_bridge_id(BridgeId),
-    remove(BridgeType, BridgeName, #{}).
+    remove(BridgeType, BridgeName, #{}, #{}).
 
 remove(Type, Name) ->
-    remove(Type, Name, undefined).
+    remove(Type, Name, #{}, #{}).
 
 %% just for perform_bridge_changes/1
-remove(Type, Name, _Conf) ->
+remove(Type, Name, _Conf, _Opts) ->
     ?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}),
     case emqx_resource:remove_local(resource_id(Type, Name)) of
         ok -> ok;
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index a3fb6c402..abdb220bb 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -1,32 +1,4 @@
 emqx_resource_schema {
-  batch {
-    desc {
-      en: """
-Configuration of batch query.</br>
-Batch requests are made immediately when the number of requests reaches the `batch_size`, or also immediately when the number of requests is less than the batch request size but the maximum batch_time has been reached.
-"""
-      zh: """
-批量请求配置。</br>
-请求数达到批量请求大小时立刻进行批量请求，或当请求数不足批量请求数大小，但已经达到最大批量等待时间时也立即进行批量请求。
-"""
-    }
-    label {
-      en: """batch"""
-      zh: """批量请求"""
-    }
-  }
-
-  queue {
-    desc {
-      en: """Configuration of queue."""
-      zh: """请求队列配置"""
-    }
-    label {
-      en: """queue"""
-      zh: """请求队列"""
-    }
-  }
-
   query_mode {
     desc {
       en: """Query mode. Optional 'sync/async', default 'sync'."""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 5327e3aae..5b6856dc0 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -40,7 +40,7 @@
     metrics := emqx_metrics_worker:metrics()
 }.
 -type resource_group() :: binary().
--type create_opts() :: #{
+-type creation_opts() :: #{
     health_check_interval => integer(),
     health_check_timeout => integer(),
     %% We can choose to block the return of emqx_resource:start until
@@ -52,7 +52,15 @@
     start_after_created => boolean(),
     %% If the resource disconnected, we can set to retry starting the resource
     %% periodically.
-    auto_retry_interval => integer()
+    auto_retry_interval => integer(),
+    enable_batch => boolean(),
+    batch_size => integer(),
+    batch_time => integer(),
+    enable_queue => boolean(),
+    queue_max_bytes => integer(),
+    query_mode => async | sync | dynamic
+    resume_interval => integer(),
+    async_inflight_window => integer()
 }.
 -type query_result() ::
     ok
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 6601b9eea..b134d2af1 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -102,6 +102,7 @@
     list_instances_verbose/0,
     %% return the data of the instance
     get_instance/1,
+    fetch_creation_opts/1,
     %% return all the instances of the same resource type
     list_instances_by_type/1,
     generate_id/1,
@@ -159,7 +160,7 @@ is_resource_mod(Module) ->
 create(ResId, Group, ResourceType, Config) ->
     create(ResId, Group, ResourceType, Config, #{}).
 
--spec create(resource_id(), resource_group(), resource_type(), resource_config(), create_opts()) ->
+-spec create(resource_id(), resource_group(), resource_type(), resource_config(), creation_opts()) ->
     {ok, resource_data() | 'already_created'} | {error, Reason :: term()}.
 create(ResId, Group, ResourceType, Config, Opts) ->
     emqx_resource_proto_v1:create(ResId, Group, ResourceType, Config, Opts).
@@ -175,7 +176,7 @@ create_local(ResId, Group, ResourceType, Config) ->
     resource_group(),
     resource_type(),
     resource_config(),
-    create_opts()
+    creation_opts()
 ) ->
     {ok, resource_data()}.
 create_local(ResId, Group, ResourceType, Config, Opts) ->
@@ -196,7 +197,7 @@ create_dry_run_local(ResourceType, Config) ->
 recreate(ResId, ResourceType, Config) ->
     recreate(ResId, ResourceType, Config, #{}).
 
--spec recreate(resource_id(), resource_type(), resource_config(), create_opts()) ->
+-spec recreate(resource_id(), resource_type(), resource_config(), creation_opts()) ->
     {ok, resource_data()} | {error, Reason :: term()}.
 recreate(ResId, ResourceType, Config, Opts) ->
     emqx_resource_proto_v1:recreate(ResId, ResourceType, Config, Opts).
@@ -206,7 +207,7 @@ recreate(ResId, ResourceType, Config, Opts) ->
 recreate_local(ResId, ResourceType, Config) ->
     recreate_local(ResId, ResourceType, Config, #{}).
 
--spec recreate_local(resource_id(), resource_type(), resource_config(), create_opts()) ->
+-spec recreate_local(resource_id(), resource_type(), resource_config(), creation_opts()) ->
     {ok, resource_data()} | {error, Reason :: term()}.
 recreate_local(ResId, ResourceType, Config, Opts) ->
     emqx_resource_manager:recreate(ResId, ResourceType, Config, Opts).
@@ -249,7 +250,7 @@ simple_async_query(ResId, Request, ReplyFun) ->
 start(ResId) ->
     start(ResId, #{}).
 
--spec start(resource_id(), create_opts()) -> ok | {error, Reason :: term()}.
+-spec start(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}.
 start(ResId, Opts) ->
     emqx_resource_manager:start(ResId, Opts).
 
@@ -257,7 +258,7 @@ start(ResId, Opts) ->
 restart(ResId) ->
     restart(ResId, #{}).
 
--spec restart(resource_id(), create_opts()) -> ok | {error, Reason :: term()}.
+-spec restart(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}.
 restart(ResId, Opts) ->
     emqx_resource_manager:restart(ResId, Opts).
 
@@ -277,6 +278,25 @@ set_resource_status_connecting(ResId) ->
 get_instance(ResId) ->
     emqx_resource_manager:lookup(ResId).
 
+-spec fetch_creation_opts(map()) -> creation_opts().
+fetch_creation_opts(Opts) ->
+    SupportedOpts = [
+        health_check_interval,
+        health_check_timeout,
+        wait_for_resource_ready,
+        start_after_created,
+        auto_retry_interval,
+        enable_batch,
+        batch_size,
+        batch_time,
+        enable_queue,
+        queue_max_bytes,
+        query_mode,
+        resume_interval,
+        async_inflight_window
+    ],
+    maps:with(creation_opts(), SupportedOpts).
+
 -spec list_instances() -> [resource_id()].
 list_instances() ->
     [Id || #{id := Id} <- list_instances_verbose()].
@@ -341,7 +361,7 @@ check_and_create(ResId, Group, ResourceType, RawConfig) ->
     resource_group(),
     resource_type(),
     raw_resource_config(),
-    create_opts()
+    creation_opts()
 ) ->
     {ok, resource_data() | 'already_created'} | {error, term()}.
 check_and_create(ResId, Group, ResourceType, RawConfig, Opts) ->
@@ -366,7 +386,7 @@ check_and_create_local(ResId, Group, ResourceType, RawConfig) ->
     resource_group(),
     resource_type(),
     raw_resource_config(),
-    create_opts()
+    creation_opts()
 ) -> {ok, resource_data()} | {error, term()}.
 check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) ->
     check_and_do(
@@ -379,7 +399,7 @@ check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) ->
     resource_id(),
     resource_type(),
     raw_resource_config(),
-    create_opts()
+    creation_opts()
 ) ->
     {ok, resource_data()} | {error, term()}.
 check_and_recreate(ResId, ResourceType, RawConfig, Opts) ->
@@ -393,7 +413,7 @@ check_and_recreate(ResId, ResourceType, RawConfig, Opts) ->
     resource_id(),
     resource_type(),
     raw_resource_config(),
-    create_opts()
+    creation_opts()
 ) ->
     {ok, resource_data()} | {error, term()}.
 check_and_recreate_local(ResId, ResourceType, RawConfig, Opts) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 3310555d1..608548898 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -85,7 +85,7 @@ manager_id_to_resource_id(MgrId) ->
     resource_group(),
     resource_type(),
     resource_config(),
-    create_opts()
+    creation_opts()
 ) -> {ok, resource_data()}.
 ensure_resource(ResId, Group, ResourceType, Config, Opts) ->
     case lookup(ResId) of
@@ -97,7 +97,7 @@ ensure_resource(ResId, Group, ResourceType, Config, Opts) ->
     end.
 
 %% @doc Called from emqx_resource when recreating a resource which may or may not exist
--spec recreate(resource_id(), resource_type(), resource_config(), create_opts()) ->
+-spec recreate(resource_id(), resource_type(), resource_config(), creation_opts()) ->
     {ok, resource_data()} | {error, not_found} | {error, updating_to_incorrect_resource_type}.
 recreate(ResId, ResourceType, NewConfig, Opts) ->
     case lookup(ResId) of
@@ -166,7 +166,7 @@ remove(ResId, ClearMetrics) when is_binary(ResId) ->
     safe_call(ResId, {remove, ClearMetrics}, ?T_OPERATION).
 
 %% @doc Stops and then starts an instance that was already running
--spec restart(resource_id(), create_opts()) -> ok | {error, Reason :: term()}.
+-spec restart(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}.
 restart(ResId, Opts) when is_binary(ResId) ->
     case safe_call(ResId, restart, ?T_OPERATION) of
         ok ->
@@ -177,7 +177,7 @@ restart(ResId, Opts) when is_binary(ResId) ->
     end.
 
 %% @doc Start the resource
--spec start(resource_id(), create_opts()) -> ok | {error, Reason :: term()}.
+-spec start(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}.
 start(ResId, Opts) ->
     case safe_call(ResId, start, ?T_OPERATION) of
         ok ->
diff --git a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl
index cdd2592d9..11af1a62c 100644
--- a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl
+++ b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl
@@ -38,7 +38,7 @@ introduced_in() ->
     resource_group(),
     resource_type(),
     resource_config(),
-    create_opts()
+    creation_opts()
 ) ->
     {ok, resource_data() | 'already_created'} | {error, Reason :: term()}.
 create(ResId, Group, ResourceType, Config, Opts) ->
@@ -58,7 +58,7 @@ create_dry_run(ResourceType, Config) ->
     resource_id(),
     resource_type(),
     resource_config(),
-    create_opts()
+    creation_opts()
 ) ->
     {ok, resource_data()} | {error, Reason :: term()}.
 recreate(ResId, ResourceType, Config, Opts) ->
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index 933cd0189..464c055b7 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -30,22 +30,14 @@ namespace() -> "resource_schema".
 
 roots() -> [].
 
-fields('batch&async&queue') ->
+fields('creation_opts') ->
     [
         {query_mode, fun query_mode/1},
         {resume_interval, fun resume_interval/1},
         {async_inflight_window, fun async_inflight_window/1},
-        {batch, mk(ref(?MODULE, batch), #{desc => ?DESC("batch")})},
-        {queue, mk(ref(?MODULE, queue), #{desc => ?DESC("queue")})}
-    ];
-fields(batch) ->
-    [
         {enable_batch, fun enable_batch/1},
         {batch_size, fun batch_size/1},
-        {batch_time, fun batch_time/1}
-    ];
-fields(queue) ->
-    [
+        {batch_time, fun batch_time/1},
         {enable_queue, fun enable_queue/1},
         {max_queue_bytes, fun queue_max_bytes/1}
     ].
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 4edeb786a..23c6788e8 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -111,7 +111,7 @@ fields(Name) when
     Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2
 ->
     fields(basic) ++
-        emqx_resource_schema:fields('batch&async&queue') ++
+        emqx_resource_schema:fields('creation_opts') ++
         connector_field(Name).
 
 method_fileds(post, ConnectorType) ->

From 3a76a50382a55ddc1b394bd9d3d671c56b0e2968 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 19:45:52 +0800
Subject: [PATCH 052/232] fix: syntax error and compile error

---
 apps/emqx_bridge/src/emqx_bridge_resource.erl         | 2 +-
 apps/emqx_resource/include/emqx_resource.hrl          | 2 +-
 apps/emqx_resource/src/emqx_resource.erl              | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 4 +++-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index ac1ec6ba3..66c4524b0 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -34,7 +34,7 @@
     create_dry_run/2,
     remove/1,
     remove/2,
-    remove/3,
+    remove/4,
     update/2,
     update/3,
     stop/2,
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 5b6856dc0..ed8929831 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -58,7 +58,7 @@
     batch_time => integer(),
     enable_queue => boolean(),
     queue_max_bytes => integer(),
-    query_mode => async | sync | dynamic
+    query_mode => async | sync | dynamic,
     resume_interval => integer(),
     async_inflight_window => integer()
 }.
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index b134d2af1..c4fd24007 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -295,7 +295,7 @@ fetch_creation_opts(Opts) ->
         resume_interval,
         async_inflight_window
     ],
-    maps:with(creation_opts(), SupportedOpts).
+    maps:with(SupportedOpts, Opts).
 
 -spec list_instances() -> [resource_id()].
 list_instances() ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 23c6788e8..ece2a6bc7 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -71,7 +71,9 @@ values(Protocol, post) ->
             <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
                 "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary,
                 "bool=${payload.bool}">>,
-        batch => #{enable_batch => false, batch_size => 5, batch_time => <<"1m">>}
+        enable_batch => false,
+        batch_size => 5,
+        batch_time => <<"1m">>
     };
 values(Protocol, put) ->
     values(Protocol, post).

From db3e4f0b906aa811c6fec268381f15fc9a2ff2ec Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 20:58:56 +0800
Subject: [PATCH 053/232] fix: flatten Points list

---
 .../emqx_ee_connector/src/emqx_ee_connector_influxdb.erl  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 41f7059ec..0a2bbb638 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -389,13 +389,13 @@ to_maps_config(K, V, Res) ->
 %% Tags & Fields Data Trans
 parse_batch_data(InstId, BatchData, SyntaxLines) ->
     {Points, Errors} = lists:foldl(
-        fun({send_message, Data}, {AccIn, ErrAccIn}) ->
+        fun({send_message, Data}, {ListOfPoints, ErrAccIn}) ->
             case data_to_points(Data, SyntaxLines) of
                 {ok, Points} ->
-                    {[Points | AccIn], ErrAccIn};
+                    {[Points | ListOfPoints], ErrAccIn};
                 {error, ErrorPoints} ->
                     log_error_points(InstId, ErrorPoints),
-                    {AccIn, ErrAccIn + 1}
+                    {ListOfPoints, ErrAccIn + 1}
             end
         end,
         {[], 0},
@@ -403,7 +403,7 @@ parse_batch_data(InstId, BatchData, SyntaxLines) ->
     ),
     case Errors of
         0 ->
-            {ok, Points};
+            {ok, lists:flatten(Points)};
         _ ->
             ?SLOG(error, #{
                 msg => io_lib:format("InfluxDB trans point failed, count: ~p", [Errors]),

From 88388b0c54dabb38ad3e55ae9a540652705b7fd7 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 11 Aug 2022 18:13:27 +0800
Subject: [PATCH 054/232] feat: http connector support async sending

---
 .../{ => schema}/emqx_bridge_mqtt_schema.erl  |  0
 .../src/{ => schema}/emqx_bridge_schema.erl   | 14 ++---
 .../emqx_bridge_webhook_schema.erl            |  0
 .../src/emqx_connector_http.erl               | 52 +++++++++++++++++--
 mix.exs                                       |  2 +-
 rebar.config                                  |  2 +-
 6 files changed, 58 insertions(+), 12 deletions(-)
 rename apps/emqx_bridge/src/{ => schema}/emqx_bridge_mqtt_schema.erl (100%)
 rename apps/emqx_bridge/src/{ => schema}/emqx_bridge_schema.erl (96%)
 rename apps/emqx_bridge/src/{ => schema}/emqx_bridge_webhook_schema.erl (100%)

diff --git a/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
similarity index 100%
rename from apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl
rename to apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
similarity index 96%
rename from apps/emqx_bridge/src/emqx_bridge_schema.erl
rename to apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 46039390d..4343dc223 100644
--- a/apps/emqx_bridge/src/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -27,15 +27,15 @@
 %%======================================================================================
 %% For HTTP APIs
 get_response() ->
-    http_schema("get").
+    api_schema("get").
 
 put_request() ->
-    http_schema("put").
+    api_schema("put").
 
 post_request() ->
-    http_schema("post").
+    api_schema("post").
 
-http_schema(Method) ->
+api_schema(Method) ->
     Broker =
         lists:flatmap(
             fun(Type) ->
@@ -46,17 +46,17 @@ http_schema(Method) ->
             end,
             ?CONN_TYPES
         ) ++ [ref(Module, Method) || Module <- [emqx_bridge_webhook_schema]],
-    EE = ee_schemas(Method),
+    EE = ee_api_schemas(Method),
     hoconsc:union(Broker ++ EE).
 
 -if(?EMQX_RELEASE_EDITION == ee).
-ee_schemas(Method) ->
+ee_api_schemas(Method) ->
     emqx_ee_bridge:api_schemas(Method).
 
 ee_fields_bridges() ->
     emqx_ee_bridge:fields(bridges).
 -else.
-ee_schemas(_) ->
+ee_api_schemas(_) ->
     [].
 
 ee_fields_bridges() ->
diff --git a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
similarity index 100%
rename from apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl
rename to apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index c5a1b89db..11030fef6 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -30,6 +30,7 @@
     on_start/2,
     on_stop/2,
     on_query/3,
+    on_query_async/4,
     on_get_status/2
 ]).
 
@@ -165,7 +166,7 @@ ref(Field) -> hoconsc:ref(?MODULE, Field).
 
 %% ===================================================================
 
-callback_mode() -> always_sync.
+callback_mode() -> async_if_possible.
 
 on_start(
     InstId,
@@ -231,7 +232,8 @@ on_stop(InstId, #{pool_name := PoolName}) ->
 on_query(InstId, {send_message, Msg}, State) ->
     case maps:get(request, State, undefined) of
         undefined ->
-            ?SLOG(error, #{msg => "request_not_found", connector => InstId});
+            ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}),
+            {error, arg_request_not_found};
         Request ->
             #{
                 method := Method,
@@ -302,6 +304,51 @@ on_query(
     end,
     Result.
 
+on_query_async(InstId, {send_message, Msg}, ReplyFun, State) ->
+    case maps:get(request, State, undefined) of
+        undefined ->
+            ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}),
+            {error, arg_request_not_found};
+        Request ->
+            #{
+                method := Method,
+                path := Path,
+                body := Body,
+                headers := Headers,
+                request_timeout := Timeout
+            } = process_request(Request, Msg),
+            on_query_async(
+                InstId,
+                {undefined, Method, {Path, Headers, Body}, Timeout},
+                ReplyFun,
+                State
+            )
+    end;
+on_query_async(
+    InstId,
+    {KeyOrNum, Method, Request, Timeout},
+    ReplyFun,
+    #{pool_name := PoolName, base_path := BasePath} = State
+) ->
+    ?TRACE(
+        "QUERY_ASYNC",
+        "http_connector_received",
+        #{request => Request, connector => InstId, state => State}
+    ),
+    NRequest = formalize_request(Method, BasePath, Request),
+    Worker =
+        case KeyOrNum of
+            undefined -> ehttpc_pool:pick_worker(PoolName);
+            _ -> ehttpc_pool:pick_worker(PoolName, KeyOrNum)
+        end,
+    ok = ehttpc:request_async(
+        Worker,
+        Method,
+        NRequest,
+        Timeout,
+        ReplyFun
+    ).
+
 on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) ->
     case do_get_status(PoolName, Timeout) of
         true ->
@@ -343,7 +390,6 @@ do_get_status(PoolName, Timeout) ->
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
-
 preprocess_request(undefined) ->
     undefined;
 preprocess_request(Req) when map_size(Req) == 0 ->
diff --git a/mix.exs b/mix.exs
index 5209de843..2e4765ce1 100644
--- a/mix.exs
+++ b/mix.exs
@@ -47,7 +47,7 @@ defmodule EMQXUmbrella.MixProject do
       {:lc, github: "emqx/lc", tag: "0.3.1"},
       {:redbug, "2.0.7"},
       {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
-      {:ehttpc, github: "emqx/ehttpc", tag: "0.3.0", override: true},
+      {:ehttpc, github: "emqx/ehttpc", tag: "0.4.0", override: true},
       {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
       {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
diff --git a/rebar.config b/rebar.config
index 3d3d5968f..bc5c8d396 100644
--- a/rebar.config
+++ b/rebar.config
@@ -49,7 +49,7 @@
     , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}}
     , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.7"}}}
-    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.3.0"}}}
+    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.0"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}

From ed796acb9558ca7ec74b203c7a42903f10d04933 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 22:11:28 +0800
Subject: [PATCH 055/232] fix: fetch resource options after reboot

---
 apps/emqx_bridge/src/emqx_bridge.erl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index c794a25a5..08939d915 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -49,14 +49,14 @@
 -export([get_basic_usage_info/0]).
 
 load() ->
-    %% set wait_for_resource_ready => 0 to start resources async
-    Opts = #{auto_retry_interval => 60000, wait_for_resource_ready => 0},
     Bridges = emqx:get_config([bridges], #{}),
     lists:foreach(
         fun({Type, NamedConf}) ->
             lists:foreach(
                 fun({Name, Conf}) ->
-                    safe_load_bridge(Type, Name, Conf, Opts)
+                    %% fetch opts for `emqx_resource_worker`
+                    ResOpts = emqx_resource:fetch_creation_opts(Conf),
+                    safe_load_bridge(Type, Name, Conf, ResOpts)
                 end,
                 maps:to_list(NamedConf)
             )

From 458dab53c5434fe2c9e034e48b100e2c8793608a Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 22:54:32 +0800
Subject: [PATCH 056/232] fix: undefined_functions dialyzer warning

---
 apps/emqx_bridge/src/emqx_bridge_resource.erl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index 66c4524b0..d34c30ee9 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -37,6 +37,7 @@
     remove/4,
     update/2,
     update/3,
+    update/4,
     stop/2,
     restart/2,
     reset_metrics/1
@@ -111,6 +112,9 @@ update(BridgeId, {OldConf, Conf}) ->
     update(BridgeType, BridgeName, {OldConf, Conf}).
 
 update(Type, Name, {OldConf, Conf}) ->
+    update(Type, Name, {OldConf, Conf}, #{}).
+
+update(Type, Name, {OldConf, Conf}, Opts) ->
     %% TODO: sometimes its not necessary to restart the bridge connection.
     %%
     %% - if the connection related configs like `servers` is updated, we should restart/start
@@ -127,7 +131,7 @@ update(Type, Name, {OldConf, Conf}) ->
                 name => Name,
                 config => Conf
             }),
-            case recreate(Type, Name, Conf) of
+            case recreate(Type, Name, Conf, Opts) of
                 {ok, _} ->
                     maybe_disable_bridge(Type, Name, Conf);
                 {error, not_found} ->
@@ -137,7 +141,7 @@ update(Type, Name, {OldConf, Conf}) ->
                         name => Name,
                         config => Conf
                     }),
-                    create(Type, Name, Conf);
+                    create(Type, Name, Conf, Opts);
                 {error, Reason} ->
                     {error, {update_bridge_failed, Reason}}
             end;

From 83746daad529e08131774f795e04b392d6ef5ce8 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 11 Aug 2022 22:59:27 +0800
Subject: [PATCH 057/232] fix: update bridge config badmap error

---
 apps/emqx_bridge/src/emqx_bridge.erl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index 08939d915..354c4faee 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -260,6 +260,13 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) ->
         fun
             ({_Type, _Name}, _Conf, {error, Reason}) ->
                 {error, Reason};
+            %% for emqx_bridge_resource:update/4
+            ({Type, Name}, {OldConf, Conf}, _) ->
+                ResOpts = emqx_resource:fetch_creation_opts(Conf),
+                case Action(Type, Name, {OldConf, Conf}, ResOpts) of
+                    {error, Reason} -> {error, Reason};
+                    Return -> Return
+                end;
             ({Type, Name}, Conf, _) ->
                 ResOpts = emqx_resource:fetch_creation_opts(Conf),
                 case Action(Type, Name, Conf, ResOpts) of

From c3c4ed02b4d4840f9b9081b21c0757eb1562dedb Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 12 Aug 2022 00:24:58 +0800
Subject: [PATCH 058/232] fix: bump emqx_dashboard to 5.0.4

---
 apps/emqx_dashboard/src/emqx_dashboard.app.src         | 2 +-
 apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index c3b7b4f13..4e1a3518f 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.3"},
+    {vsn, "5.0.4"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index abdb220bb..9a2234ff5 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -45,7 +45,7 @@ emqx_resource_schema {
 
   async_inflight_window {
     desc {
-      en: """Async queyr inflight window."""
+      en: """Async query inflight window."""
       zh: """异步请求飞行队列窗口大小"""
     }
     label {

From bed10b097f7a43e1a120985907de7dbcbecef62b Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 12 Aug 2022 00:27:13 +0800
Subject: [PATCH 059/232] chore: release e5.0.0-beta.1

---
 apps/emqx/include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index 75d5852c9..bad7e8229 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -35,7 +35,7 @@
 -define(EMQX_RELEASE_CE, "5.0.4").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.0.0-alpha.1").
+-define(EMQX_RELEASE_EE, "5.0.0-beta.1").
 
 %% the HTTP API version
 -define(EMQX_API_VERSION, "5.0").

From 54298183fba32b5ef44f93da4a20ed772aa31559 Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Fri, 12 Aug 2022 09:50:50 +0800
Subject: [PATCH 060/232] ci: do not push enterprise image to aws ecr

---
 .github/workflows/build_and_push_docker_images.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index c54ae7bd9..1b724d9e3 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -121,6 +121,8 @@ jobs:
           - aws-arm64
           - ubuntu-20.04
         exclude:
+          - registry: 'public.ecr.aws'
+            profile: emqx-enterprise
           - arch: arm64
             build_machine: ubuntu-20.04
           - arch: amd64

From 0c1285cdab56098e606211d21f60675df42ad0b4 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 12 Aug 2022 10:07:54 +0800
Subject: [PATCH 061/232] fix: bridge creation opts refs

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index ece2a6bc7..6a1b2677f 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -90,7 +90,8 @@ fields(basic) ->
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
         {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
         {write_syntax, fun write_syntax/1}
-    ];
+    ] ++
+        emqx_resource_schema:fields('creation_opts');
 fields("post_udp") ->
     method_fileds(post, influxdb_udp);
 fields("post_api_v1") ->
@@ -113,7 +114,6 @@ fields(Name) when
     Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2
 ->
     fields(basic) ++
-        emqx_resource_schema:fields('creation_opts') ++
         connector_field(Name).
 
 method_fileds(post, ConnectorType) ->

From 907d9a166acdd9a0b6f8b6db493fbfcab312f207 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Fri, 12 Aug 2022 10:20:15 +0800
Subject: [PATCH 062/232] test(gw): fix gateway flaky tests

---
 apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl | 2 +-
 apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl
index 6bbd2135b..4dd5819b3 100644
--- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl
@@ -79,7 +79,7 @@ end_per_suite(Config) ->
     emqx_gateway_auth_ct:stop(),
     ok = emqx_authz_test_lib:restore_authorizers(),
     emqx_config:erase(gateway),
-    emqx_mgmt_api_test_util:end_suite([cowboy, emqx_authz, emqx_authn, emqx_gateway]),
+    emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_authz, emqx_conf]),
     Config.
 
 init_per_testcase(_Case, Config) ->
diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl
index 54928a792..084fd9f57 100644
--- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl
@@ -103,7 +103,7 @@ init_per_suite(Config) ->
 
 end_per_suite(_) ->
     {ok, _} = emqx:remove_config([gateway, mqttsn]),
-    emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_auhtn, emqx_conf]).
+    emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]).
 
 restart_mqttsn_with_subs_resume_on() ->
     Conf = emqx:get_raw_config([gateway, mqttsn]),

From 0cdf4b47f11264389a283b5ea164b1d0f35e352b Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 12 Aug 2022 11:33:19 +0800
Subject: [PATCH 063/232] feat: add more resource creation opts

---
 apps/emqx_authn/src/emqx_authn_utils.erl      |  1 -
 apps/emqx_authz/src/emqx_authz_utils.erl      |  1 -
 apps/emqx_bridge/src/emqx_bridge_resource.erl |  4 +-
 .../src/schema/emqx_bridge_webhook_schema.erl | 18 +++++-
 .../i18n/emqx_resource_schema_i18n.conf       | 56 ++++++++++++++++++-
 apps/emqx_resource/include/emqx_resource.hrl  | 33 ++++++-----
 apps/emqx_resource/src/emqx_resource.erl      |  5 +-
 .../src/emqx_resource_manager.erl             | 25 ++++-----
 .../src/schema/emqx_resource_schema.erl       | 28 ++++++++++
 9 files changed, 136 insertions(+), 35 deletions(-)

diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl
index 994f2f275..dca243cc1 100644
--- a/apps/emqx_authn/src/emqx_authn_utils.erl
+++ b/apps/emqx_authn/src/emqx_authn_utils.erl
@@ -47,7 +47,6 @@
 ]).
 
 -define(DEFAULT_RESOURCE_OPTS, #{
-    auto_retry_interval => 6000,
     start_after_created => false
 }).
 
diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl
index dd6f66c7f..0048c0dc4 100644
--- a/apps/emqx_authz/src/emqx_authz_utils.erl
+++ b/apps/emqx_authz/src/emqx_authz_utils.erl
@@ -40,7 +40,6 @@
 ]).
 
 -define(DEFAULT_RESOURCE_OPTS, #{
-    auto_retry_interval => 6000,
     start_after_created => false
 }).
 
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index d34c30ee9..f7aeec30d 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -89,7 +89,7 @@ create(BridgeId, Conf) ->
     create(BridgeType, BridgeName, Conf).
 
 create(Type, Name, Conf) ->
-    create(Type, Name, Conf, #{auto_retry_interval => 60000}).
+    create(Type, Name, Conf, #{}).
 
 create(Type, Name, Conf, Opts) ->
     ?SLOG(info, #{
@@ -169,7 +169,7 @@ recreate(Type, Name, Conf, Opts) ->
         resource_id(Type, Name),
         bridge_to_resource_type(Type),
         parse_confs(Type, Name, Conf),
-        Opts#{auto_retry_interval => 60000}
+        Opts
     ).
 
 create_dry_run(Type, Conf) ->
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
index d833e6ca8..d51c2edef 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
@@ -50,7 +50,7 @@ basic_config() ->
                     default => egress
                 }
             )}
-    ] ++
+    ] ++ webhook_creation_opts() ++
         proplists:delete(base_url, emqx_connector_http:fields(config)).
 
 request_config() ->
@@ -116,6 +116,22 @@ request_config() ->
             )}
     ].
 
+webhook_creation_opts() ->
+    Opts = emqx_resource_schema:fields(creation_opts),
+    lists:filter(
+        fun({K, _V}) ->
+            not lists:member(K, unsupported_opts())
+        end,
+        Opts
+    ).
+
+unsupported_opts() ->
+    [
+        enable_batch,
+        batch_size,
+        batch_time
+    ].
+
 %%======================================================================================
 
 type_field() ->
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index 9a2234ff5..3ec170ebf 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -1,4 +1,58 @@
 emqx_resource_schema {
+
+  health_check_interval {
+    desc {
+      en: """Health check interval, in milliseconds."""
+      zh: """健康检查间隔，单位毫秒。"""
+    }
+    label {
+      en: """Health Check Interval"""
+      zh: """健康检查间隔"""
+    }
+  }
+
+  start_after_created {
+    desc {
+      en: """Whether start the resource right after created."""
+      zh: """是否在创建资源后立即启动资源。"""
+    }
+    label {
+      en: """Start After Created"""
+      zh: """创建后立即启动"""
+    }
+  }
+
+  start_timeout {
+    desc {
+      en: """
+If 'start_after_created' enabled, how long time do we wait for the
+resource get started, in milliseconds.
+"""
+      zh: """
+如果选择了创建后立即启动资源，此选项用来设置等待资源启动的超时时间，单位毫秒。
+"""
+    }
+    label {
+      en: """Start Timeout"""
+      zh: """启动超时时间"""
+    }
+  }
+
+  auto_restart_interval {
+    desc {
+      en: """
+The auto restart interval after the resource is disconnected, in milliseconds.
+"""
+      zh: """
+资源断开以后，自动重连的时间间隔，单位毫秒。
+"""
+    }
+    label {
+      en: """Auto Restart Interval"""
+      zh: """自动重连间隔"""
+    }
+  }
+
   query_mode {
     desc {
       en: """Query mode. Optional 'sync/async', default 'sync'."""
@@ -6,7 +60,7 @@ emqx_resource_schema {
     }
     label {
       en: """query_mode"""
-      zh: """query_mode"""
+      zh: """请求模式"""
     }
   }
 
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index ed8929831..190c278ae 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -41,18 +41,25 @@
 }.
 -type resource_group() :: binary().
 -type creation_opts() :: #{
-    health_check_interval => integer(),
+    %%======================================= Deprecated Opts:
+    %% use health_check_interval instead
     health_check_timeout => integer(),
-    %% We can choose to block the return of emqx_resource:start until
-    %% the resource connected, wait max to `wait_for_resource_ready` ms.
+    %% use start_timeout instead
     wait_for_resource_ready => integer(),
+    %% use auto_restart_interval instead
+    auto_retry_interval => integer(),
+    %%======================================= Deprecated Opts End
+    health_check_interval => integer(),
+    %% We can choose to block the return of emqx_resource:start until
+    %% the resource connected, wait max to `start_timeout` ms.
+    start_timeout => integer(),
     %% If `start_after_created` is set to true, the resource is started right
     %% after it is created. But note that a `started` resource is not guaranteed
     %% to be `connected`.
     start_after_created => boolean(),
     %% If the resource disconnected, we can set to retry starting the resource
     %% periodically.
-    auto_retry_interval => integer(),
+    auto_restart_interval => integer(),
     enable_batch => boolean(),
     batch_size => integer(),
     batch_time => integer(),
@@ -68,17 +75,17 @@
     | {error, term()}
     | {resource_down, term()}.
 
-%% count
--define(DEFAULT_BATCH_SIZE, 100).
-%% milliseconds
--define(DEFAULT_BATCH_TIME, 10).
-
-%% bytes
 -define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
-
+-define(DEFAULT_BATCH_SIZE, 100).
+-define(DEFAULT_BATCH_TIME, 10).
 -define(DEFAULT_INFLIGHT, 100).
-
+-define(HEALTHCHECK_INTERVAL, 15000).
+-define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>).
 -define(RESUME_INTERVAL, 15000).
-
+-define(START_AFTER_CREATED, true).
+-define(START_TIMEOUT, 5000).
+-define(START_TIMEOUT_RAW, <<"5s">>).
+-define(AUTO_RESTART_INTERVAL, 60000).
+-define(AUTO_RESTART_INTERVAL_RAW, <<"60s">>).
 -define(TEST_ID_PREFIX, "_test_:").
 -define(RES_METRICS, resource_metrics).
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index c4fd24007..60f0dd360 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -282,10 +282,9 @@ get_instance(ResId) ->
 fetch_creation_opts(Opts) ->
     SupportedOpts = [
         health_check_interval,
-        health_check_timeout,
-        wait_for_resource_ready,
+        start_timeout,
         start_after_created,
-        auto_retry_interval,
+        auto_restart_interval,
         enable_batch,
         batch_size,
         batch_time,
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 608548898..66d9e32b0 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -57,7 +57,6 @@
 -type data() :: #data{}.
 
 -define(SHORT_HEALTHCHECK_INTERVAL, 1000).
--define(HEALTHCHECK_INTERVAL, 15000).
 -define(ETS_TABLE, ?MODULE).
 -define(WAIT_FOR_RESOURCE_DELAY, 100).
 -define(T_OPERATION, 5000).
@@ -127,9 +126,9 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         [matched]
     ),
     ok = emqx_resource_worker_sup:start_workers(ResId, Opts),
-    case maps:get(start_after_created, Opts, true) of
+    case maps:get(start_after_created, Opts, ?START_AFTER_CREATED) of
         true ->
-            wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000));
+            wait_for_ready(ResId, maps:get(start_timeout, Opts, ?START_TIMEOUT));
         false ->
             ok
     end,
@@ -147,7 +146,7 @@ create_dry_run(ResourceType, Config) ->
     ok = emqx_resource_manager_sup:ensure_child(
         MgrId, ResId, <<"dry_run">>, ResourceType, Config, #{}
     ),
-    case wait_for_resource_ready(ResId, 15000) of
+    case wait_for_ready(ResId, 15000) of
         ok ->
             remove(ResId);
         timeout ->
@@ -170,7 +169,7 @@ remove(ResId, ClearMetrics) when is_binary(ResId) ->
 restart(ResId, Opts) when is_binary(ResId) ->
     case safe_call(ResId, restart, ?T_OPERATION) of
         ok ->
-            wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)),
+            wait_for_ready(ResId, maps:get(start_timeout, Opts, 5000)),
             ok;
         {error, _Reason} = Error ->
             Error
@@ -181,7 +180,7 @@ restart(ResId, Opts) when is_binary(ResId) ->
 start(ResId, Opts) ->
     case safe_call(ResId, start, ?T_OPERATION) of
         ok ->
-            wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)),
+            wait_for_ready(ResId, maps:get(start_timeout, Opts, 5000)),
             ok;
         {error, _Reason} = Error ->
             Error
@@ -422,7 +421,7 @@ get_owner(ResId) ->
     end.
 
 handle_disconnected_state_enter(Data) ->
-    case maps:get(auto_retry_interval, Data#data.opts, undefined) of
+    case maps:get(auto_restart_interval, Data#data.opts, ?AUTO_RESTART_INTERVAL) of
         undefined ->
             {next_state, disconnected, Data};
         RetryInterval ->
@@ -573,19 +572,19 @@ data_record_to_external_map_with_metrics(Data) ->
         metrics => get_metrics(Data#data.id)
     }.
 
--spec wait_for_resource_ready(resource_id(), integer()) -> ok | timeout.
-wait_for_resource_ready(ResId, WaitTime) ->
-    do_wait_for_resource_ready(ResId, WaitTime div ?WAIT_FOR_RESOURCE_DELAY).
+-spec wait_for_ready(resource_id(), integer()) -> ok | timeout.
+wait_for_ready(ResId, WaitTime) ->
+    do_wait_for_ready(ResId, WaitTime div ?WAIT_FOR_RESOURCE_DELAY).
 
-do_wait_for_resource_ready(_ResId, 0) ->
+do_wait_for_ready(_ResId, 0) ->
     timeout;
-do_wait_for_resource_ready(ResId, Retry) ->
+do_wait_for_ready(ResId, Retry) ->
     case ets_lookup(ResId) of
         {ok, _Group, #{status := connected}} ->
             ok;
         _ ->
             timer:sleep(?WAIT_FOR_RESOURCE_DELAY),
-            do_wait_for_resource_ready(ResId, Retry - 1)
+            do_wait_for_ready(ResId, Retry - 1)
     end.
 
 safe_call(ResId, Message, Timeout) ->
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index 464c055b7..ccc31a707 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -32,6 +32,10 @@ roots() -> [].
 
 fields('creation_opts') ->
     [
+        {health_check_interval, fun health_check_interval/1},
+        {start_after_created, fun start_after_created/1},
+        {start_timeout, fun start_timeout/1},
+        {auto_restart_interval, fun auto_restart_interval/1},
         {query_mode, fun query_mode/1},
         {resume_interval, fun resume_interval/1},
         {async_inflight_window, fun async_inflight_window/1},
@@ -42,6 +46,30 @@ fields('creation_opts') ->
         {max_queue_bytes, fun queue_max_bytes/1}
     ].
 
+health_check_interval(type) -> emqx_schema:duration_ms();
+health_check_interval(desc) -> ?DESC("health_check_interval");
+health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW;
+health_check_interval(required) -> false;
+health_check_interval(_) -> undefined.
+
+start_after_created(type) -> boolean();
+start_after_created(required) -> false;
+start_after_created(default) -> ?START_AFTER_CREATED;
+start_after_created(desc) -> ?DESC("start_after_created");
+start_after_created(_) -> undefined.
+
+start_timeout(type) -> emqx_schema:duration_ms();
+start_timeout(desc) -> ?DESC("start_timeout");
+start_timeout(default) -> ?START_TIMEOUT_RAW;
+start_timeout(required) -> false;
+start_timeout(_) -> undefined.
+
+auto_restart_interval(type) -> hoconsc:union([infinity, emqx_schema:duration_ms()]);
+auto_restart_interval(desc) -> ?DESC("auto_restart_interval");
+auto_restart_interval(default) -> ?AUTO_RESTART_INTERVAL_RAW;
+auto_restart_interval(required) -> false;
+auto_restart_interval(_) -> undefined.
+
 query_mode(type) -> enum([sync, async]);
 query_mode(desc) -> ?DESC("query_mode");
 query_mode(default) -> sync;

From 3678673124ec72aabd3564386c0cab0374fb4a57 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 12 Aug 2022 10:44:47 +0800
Subject: [PATCH 064/232] fix: schema default value using raw type before
 convert

---
 apps/emqx_resource/include/emqx_resource.hrl  | 19 +++++++++++++++++++
 .../src/schema/emqx_resource_schema.erl       |  6 +++---
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 190c278ae..8ec57a00e 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -76,16 +76,35 @@
     | {resource_down, term()}.
 
 -define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
+-define(DEFAULT_QUEUE_SIZE_RAW, <<"1GB">>).
+
+%% count
 -define(DEFAULT_BATCH_SIZE, 100).
+
+%% milliseconds
 -define(DEFAULT_BATCH_TIME, 10).
+-define(DEFAULT_BATCH_TIME_RAW, <<"10ms">>).
+
+%% count
 -define(DEFAULT_INFLIGHT, 100).
+
+%% milliseconds
 -define(HEALTHCHECK_INTERVAL, 15000).
 -define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>).
+
+%% milliseconds
 -define(RESUME_INTERVAL, 15000).
+-define(RESUME_INTERVAL_RAW, <<"15s">>).
+
 -define(START_AFTER_CREATED, true).
+
+%% milliseconds
 -define(START_TIMEOUT, 5000).
 -define(START_TIMEOUT_RAW, <<"5s">>).
+
+%% milliseconds
 -define(AUTO_RESTART_INTERVAL, 60000).
 -define(AUTO_RESTART_INTERVAL_RAW, <<"60s">>).
+
 -define(TEST_ID_PREFIX, "_test_:").
 -define(RES_METRICS, resource_metrics).
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index ccc31a707..6111543d2 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -90,7 +90,7 @@ enable_queue(_) -> undefined.
 
 resume_interval(type) -> emqx_schema:duration_ms();
 resume_interval(desc) -> ?DESC("resume_interval");
-resume_interval(default) -> ?RESUME_INTERVAL;
+resume_interval(default) -> ?RESUME_INTERVAL_RAW;
 resume_interval(required) -> false;
 resume_interval(_) -> undefined.
 
@@ -108,12 +108,12 @@ batch_size(_) -> undefined.
 
 batch_time(type) -> emqx_schema:duration_ms();
 batch_time(desc) -> ?DESC("batch_time");
-batch_time(default) -> ?DEFAULT_BATCH_TIME;
+batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW;
 batch_time(required) -> false;
 batch_time(_) -> undefined.
 
 queue_max_bytes(type) -> emqx_schema:bytesize();
 queue_max_bytes(desc) -> ?DESC("queue_max_bytes");
-queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE;
+queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW;
 queue_max_bytes(required) -> false;
 queue_max_bytes(_) -> undefined.

From fa5e8f142291547e901e42b87b31c016fe4ba45c Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 12 Aug 2022 11:31:21 +0800
Subject: [PATCH 065/232] chore: refine i18n label

---
 .../i18n/emqx_resource_schema_i18n.conf          | 16 ++++++++--------
 .../i18n/emqx_ee_bridge_influxdb.conf            |  6 +++---
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index 3ec170ebf..aa6579bbd 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -59,7 +59,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """请求模式。可选 '同步/异步'，默认为'同步'模式。"""
     }
     label {
-      en: """query_mode"""
+      en: """Query mode"""
       zh: """请求模式"""
     }
   }
@@ -70,7 +70,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """启用批量模式。"""
     }
     label {
-      en: """enable_batch"""
+      en: """Enable batch"""
       zh: """启用批量模式"""
     }
   }
@@ -81,7 +81,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """启用队列模式。"""
     }
     label {
-      en: """enable_queue"""
+      en: """Enable queue"""
       zh: """启用队列模式"""
     }
   }
@@ -92,7 +92,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """资源不可用时的重试时间"""
     }
     label {
-      en: """resume_interval"""
+      en: """Resume interval"""
       zh: """恢复时间"""
     }
   }
@@ -103,7 +103,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """异步请求飞行队列窗口大小"""
     }
     label {
-      en: """async_inflight_window"""
+      en: """Async inflight window"""
       zh: """异步请求飞行队列窗口"""
     }
   }
@@ -114,7 +114,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """批量请求大小"""
     }
     label {
-      en: """batch_size"""
+      en: """Batch size"""
       zh: """批量请求大小"""
     }
   }
@@ -125,7 +125,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """最大批量请求等待时间。"""
     }
     label {
-      en: """batch_time"""
+      en: """Batch time"""
       zh: """批量等待间隔"""
     }
   }
@@ -136,7 +136,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
       zh: """消息队列的最大长度，以字节计。"""
     }
     label {
-      en: """queue_max_bytes"""
+      en: """Queue max bytes"""
       zh: """队列最大长度"""
     }
   }
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
index ffd0b66a0..412922408 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -40,7 +40,7 @@ TLDR:
 """
             }
         label {
-            en: "write_syntax"
+            en: "Write Syntax"
             zh: "写语句"
             }
     }
@@ -57,7 +57,7 @@ TLDR:
     config_direction {
         desc {
             en: """The direction of this bridge, MUST be 'egress'"""
-            zh: """桥接的方向， 必须是 egress"""
+            zh: """桥接的方向，必须是 egress"""
         }
         label {
             en: "Bridge Direction"
@@ -90,7 +90,7 @@ TLDR:
     desc_name {
         desc {
             en: """Bridge name, used as a human-readable description of the bridge."""
-            zh: """桥接名字，可读描述"""
+            zh: """桥接名字，人类可读的描述信息。"""
         }
         label {
             en: "Bridge Name"

From dc7953c3e10e1c0dc792389fd15c45300ad4b764 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 12 Aug 2022 17:28:56 +0800
Subject: [PATCH 066/232] chore: refine async query variable name

---
 apps/emqx_connector/src/emqx_connector_http.erl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 11030fef6..b7fd21c5f 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -304,7 +304,7 @@ on_query(
     end,
     Result.
 
-on_query_async(InstId, {send_message, Msg}, ReplyFun, State) ->
+on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) ->
     case maps:get(request, State, undefined) of
         undefined ->
             ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}),
@@ -320,14 +320,14 @@ on_query_async(InstId, {send_message, Msg}, ReplyFun, State) ->
             on_query_async(
                 InstId,
                 {undefined, Method, {Path, Headers, Body}, Timeout},
-                ReplyFun,
+                ReplyFunAndArgs,
                 State
             )
     end;
 on_query_async(
     InstId,
     {KeyOrNum, Method, Request, Timeout},
-    ReplyFun,
+    ReplyFunAndArgs,
     #{pool_name := PoolName, base_path := BasePath} = State
 ) ->
     ?TRACE(
@@ -346,7 +346,7 @@ on_query_async(
         Method,
         NRequest,
         Timeout,
-        ReplyFun
+        ReplyFunAndArgs
     ).
 
 on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) ->

From 441d8c9d5750674db210e32fd188fcd2b3c59970 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 12 Aug 2022 17:44:38 +0800
Subject: [PATCH 067/232] chore: bump `influxdb-client-erl` vsn

---
 lib-ee/emqx_ee_connector/rebar.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index 485b0120d..5963b7ab0 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, [
   {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
-  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.3"}}}
+  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.4"}}}
 ]}.
 
 {shell, [

From 594d071c05ed243e1c6fb016fdd2e73e06f5dbf2 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 12 Aug 2022 17:45:20 +0800
Subject: [PATCH 068/232] feat(influxdb): add async callback

---
 apps/emqx_resource/src/emqx_resource.erl      |  8 +++++++
 .../src/emqx_ee_connector_influxdb.erl        | 24 ++++++++++++++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 60f0dd360..b79650904 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -114,6 +114,7 @@
 -optional_callbacks([
     on_query/3,
     on_batch_query/3,
+    on_query_async/4,
     on_get_status/2
 ]).
 
@@ -130,6 +131,13 @@
 %% when calling emqx_resource:on_batch_query/3
 -callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result().
 
+-callback on_query_async(
+    resource_id(),
+    Request :: term(),
+    {ReplyFun :: function(), Args :: list()},
+    resource_state()
+) -> query_result().
+
 %% when calling emqx_resource:health_check/2
 -callback on_get_status(resource_id(), resource_state()) ->
     resource_status()
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 0a2bbb638..d0c17b6d5 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -18,6 +18,7 @@
     on_stop/2,
     on_query/3,
     on_batch_query/3,
+    on_query_async/4,
     on_get_status/2
 ]).
 
@@ -62,6 +63,20 @@ on_batch_query(InstId, BatchData, State = #{write_syntax := SyntaxLines, client
             {resource_down, disconnected}
     end.
 
+on_query_async(
+    InstId,
+    {send_message, Data},
+    {ReplayFun, Args},
+    _State = #{write_syntax := SyntaxLines, client := Client}
+) ->
+    case data_to_points(Data, SyntaxLines) of
+        {ok, Points} ->
+            do_async_query(InstId, Client, Points, {ReplayFun, Args});
+        {error, ErrorPoints} = Err ->
+            log_error_points(InstId, ErrorPoints),
+            Err
+    end.
+
 on_get_status(_InstId, #{client := Client}) ->
     case influxdb:is_alive(Client) of
         true ->
@@ -331,7 +346,6 @@ ssl_config(SSL = #{enable := true}) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% Query
-
 do_query(InstId, Client, Points) ->
     case influxdb:write(Client, Points) of
         ok ->
@@ -349,6 +363,14 @@ do_query(InstId, Client, Points) ->
             Err
     end.
 
+do_async_query(InstId, Client, Points, ReplayFunAndArgs) ->
+    ?SLOG(info, #{
+        msg => "influxdb write point async",
+        connector => InstId,
+        points => Points
+    }),
+    ok = influxdb:write_async(Client, Points, ReplayFunAndArgs).
+
 %% -------------------------------------------------------------------------------------------------
 %% Tags & Fields Config Trans
 

From b01ae8ece6a80870d538aac6362d5cff6f76faf0 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 15 Aug 2022 11:57:02 +0800
Subject: [PATCH 069/232] chore: refine influxdb bridge/connector i18n

---
 .../i18n/emqx_resource_schema_i18n.conf       | 23 +++++---------
 .../i18n/emqx_ee_bridge_influxdb.conf         | 30 ++++++++-----------
 .../i18n/emqx_ee_connector_influxdb.conf      | 16 +++++-----
 3 files changed, 28 insertions(+), 41 deletions(-)

diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index aa6579bbd..c07573b1a 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -24,13 +24,8 @@ emqx_resource_schema {
 
   start_timeout {
     desc {
-      en: """
-If 'start_after_created' enabled, how long time do we wait for the
-resource get started, in milliseconds.
-"""
-      zh: """
-如果选择了创建后立即启动资源，此选项用来设置等待资源启动的超时时间，单位毫秒。
-"""
+      en: """If 'start_after_created' enabled, how long time do we wait for the resource get started, in milliseconds."""
+      zh: """如果选择了创建后立即启动资源，此选项用来设置等待资源启动的超时时间，单位毫秒。"""
     }
     label {
       en: """Start Timeout"""
@@ -40,12 +35,8 @@ resource get started, in milliseconds.
 
   auto_restart_interval {
     desc {
-      en: """
-The auto restart interval after the resource is disconnected, in milliseconds.
-"""
-      zh: """
-资源断开以后，自动重连的时间间隔，单位毫秒。
-"""
+      en: """The auto restart interval after the resource is disconnected, in milliseconds."""
+      zh: """资源断开以后，自动重连的时间间隔，单位毫秒。"""
     }
     label {
       en: """Auto Restart Interval"""
@@ -89,7 +80,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
   resume_interval {
     desc {
       en: """Resume time interval when resource down."""
-      zh: """资源不可用时的重试时间"""
+      zh: """资源不可用时的重试时间。"""
     }
     label {
       en: """Resume interval"""
@@ -100,7 +91,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
   async_inflight_window {
     desc {
       en: """Async query inflight window."""
-      zh: """异步请求飞行队列窗口大小"""
+      zh: """异步请求飞行队列窗口大小。"""
     }
     label {
       en: """Async inflight window"""
@@ -111,7 +102,7 @@ The auto restart interval after the resource is disconnected, in milliseconds.
   batch_size {
     desc {
       en: """Maximum batch count."""
-      zh: """批量请求大小"""
+      zh: """批量请求大小。"""
     }
     label {
       en: """Batch size"""
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
index 412922408..9e805132e 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -1,15 +1,13 @@
 emqx_ee_bridge_influxdb {
     local_topic {
         desc {
-            en: """
-The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic
+            en: """The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic
 matching the local_topic will be forwarded.</br>
 NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is
 configured, then both the data got from the rule and the MQTT messages that match local_topic
 will be forwarded.
 """
-            zh: """
-发送到 'local_topic' 的消息都会转发到 InfluxDB。 </br>
+            zh: """发送到 'local_topic' 的消息都会转发到 InfluxDB。 </br>
 注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。
 """
         }
@@ -20,20 +18,18 @@ will be forwarded.
     }
     write_syntax {
         desc {
-            en: """
-Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported.
+            en: """Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported.
 See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and
 [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) </br>
-TLDR:
+TLDR:</br>
 ```
 <measurement>[,<tag_key>=<tag_value>[,<tag_key>=<tag_value>]] <field_key>=<field_value>[,<field_key>=<field_value>] [<timestamp>]
 ```
 """
-            zh: """
-使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符</br>
+            zh: """使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符</br>
 参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及
 [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) </br>
-TLDR:
+TLDR: </br>
 ```
 <measurement>[,<tag_key>=<tag_value>[,<tag_key>=<tag_value>]] <field_key>=<field_value>[,<field_key>=<field_value>] [<timestamp>]
 ```
@@ -46,8 +42,8 @@ TLDR:
     }
     config_enable {
         desc {
-            en: """Enable or disable this bridge"""
-            zh: """启用/禁用桥接"""
+            en: """Enable or disable this bridge."""
+            zh: """启用/禁用桥接。"""
         }
         label {
             en: "Enable Or Disable Bridge"
@@ -56,8 +52,8 @@ TLDR:
         }
     config_direction {
         desc {
-            en: """The direction of this bridge, MUST be 'egress'"""
-            zh: """桥接的方向，必须是 egress"""
+            en: """The direction of this bridge, MUST be 'egress'."""
+            zh: """桥接的方向，必须是 egress。"""
         }
         label {
             en: "Bridge Direction"
@@ -68,7 +64,7 @@ TLDR:
     desc_config {
         desc {
             en: """Configuration for an InfluxDB bridge."""
-            zh: """InfluxDB 桥接配置"""
+            zh: """InfluxDB 桥接配置。"""
         }
         label: {
             en: "InfluxDB Bridge Configuration"
@@ -78,8 +74,8 @@ TLDR:
 
     desc_type {
         desc {
-            en: """The Bridge Type"""
-            zh: """Bridge 类型"""
+            en: """The Bridge Type."""
+            zh: """Bridge 类型。"""
         }
         label {
             en: "Bridge Type"
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
index 7e223b9b7..4d2dc168c 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
@@ -42,8 +42,8 @@ emqx_ee_connector_influxdb {
     }
     protocol {
         desc {
-                en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2"""
-                zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2"""
+                en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2."""
+                zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2。"""
         }
         label: {
                 en: """Protocol"""
@@ -53,7 +53,7 @@ emqx_ee_connector_influxdb {
     influxdb_udp {
         desc {
                 en: """InfluxDB's UDP protocol."""
-                zh: """InfluxDB UDP 协议"""
+                zh: """InfluxDB UDP 协议。"""
         }
         label: {
                 en: """UDP Protocol"""
@@ -63,7 +63,7 @@ emqx_ee_connector_influxdb {
     influxdb_api_v1 {
         desc {
                 en: """InfluxDB's protocol. Support InfluxDB v1.8 and before."""
-                zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本"""
+                zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本。"""
         }
         label: {
                 en: """HTTP API Protocol"""
@@ -73,7 +73,7 @@ emqx_ee_connector_influxdb {
     influxdb_api_v2 {
         desc {
                 en: """InfluxDB's protocol. Support InfluxDB v2.0 and after."""
-                zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本"""
+                zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本。"""
         }
         label: {
                 en: """HTTP API V2 Protocol"""
@@ -113,7 +113,7 @@ emqx_ee_connector_influxdb {
     bucket {
         desc {
                 en: "InfluxDB bucket name."
-                zh: "InfluxDB bucket 名称"
+                zh: "InfluxDB bucket 名称。"
         }
         label: {
                 en: "Bucket"
@@ -152,8 +152,8 @@ emqx_ee_connector_influxdb {
     }
     pool_size {
         desc {
-                en: """InfluxDB Pool Size"""
-                zh: """InfluxDB 连接池大小"""
+                en: """InfluxDB Pool Size. Default value is CPU threads."""
+                zh: """InfluxDB 连接池大小，默认为 CPU 线程数。"""
             }
         label {
                 en: """InfluxDB Pool Size"""

From 68946f1f6c5638d5c41272f69484858706fdb293 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 15 Aug 2022 11:57:37 +0800
Subject: [PATCH 070/232] feat: influxdb support `async`/`batch_async` query

---
 apps/emqx_resource/src/emqx_resource.erl      | 10 +++++
 .../src/emqx_ee_connector_influxdb.erl        | 40 +++++++++++++------
 2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index b79650904..99e1f6057 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -115,6 +115,7 @@
     on_query/3,
     on_batch_query/3,
     on_query_async/4,
+    on_batch_query_async/4,
     on_get_status/2
 ]).
 
@@ -131,6 +132,7 @@
 %% when calling emqx_resource:on_batch_query/3
 -callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result().
 
+%% when calling emqx_resource:on_query_async/4
 -callback on_query_async(
     resource_id(),
     Request :: term(),
@@ -138,6 +140,14 @@
     resource_state()
 ) -> query_result().
 
+%% when calling emqx_resource:on_batch_query_async/4
+-callback on_batch_query_async(
+    resource_id(),
+    Request :: term(),
+    {ReplyFun :: function(), Args :: list()},
+    resource_state()
+) -> query_result().
+
 %% when calling emqx_resource:health_check/2
 -callback on_get_status(resource_id(), resource_state()) ->
     resource_status()
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index d0c17b6d5..2c2de9a99 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -19,6 +19,7 @@
     on_query/3,
     on_batch_query/3,
     on_query_async/4,
+    on_batch_query_async/4,
     on_get_status/2
 ]).
 
@@ -31,7 +32,7 @@
 
 %% -------------------------------------------------------------------------------------------------
 %% resource callback
-callback_mode() -> always_sync.
+callback_mode() -> async_if_possible.
 
 on_start(InstId, Config) ->
     start_client(InstId, Config).
@@ -50,17 +51,12 @@ on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, c
 
 %% Once a Batched Data trans to points failed.
 %% This batch query failed
-on_batch_query(InstId, BatchData, State = #{write_syntax := SyntaxLines, client := Client}) ->
-    case on_get_status(InstId, State) of
-        connected ->
-            case parse_batch_data(InstId, BatchData, SyntaxLines) of
-                {ok, Points} ->
-                    do_query(InstId, Client, Points);
-                {error, Reason} ->
-                    {error, Reason}
-            end;
-        disconnected ->
-            {resource_down, disconnected}
+on_batch_query(InstId, BatchData, _State = #{write_syntax := SyntaxLines, client := Client}) ->
+    case parse_batch_data(InstId, BatchData, SyntaxLines) of
+        {ok, Points} ->
+            do_query(InstId, Client, Points);
+        {error, Reason} ->
+            {error, Reason}
     end.
 
 on_query_async(
@@ -77,6 +73,24 @@ on_query_async(
             Err
     end.
 
+on_batch_query_async(
+    InstId,
+    BatchData,
+    {ReplayFun, Args},
+    State = #{write_syntax := SyntaxLines, client := Client}
+) ->
+    case on_get_status(InstId, State) of
+        connected ->
+            case parse_batch_data(InstId, BatchData, SyntaxLines) of
+                {ok, Points} ->
+                    do_async_query(InstId, Client, Points, {ReplayFun, Args});
+                {error, Reason} ->
+                    {error, Reason}
+            end;
+        disconnected ->
+            {resource_down, disconnected}
+    end.
+
 on_get_status(_InstId, #{client := Client}) ->
     case influxdb:is_alive(Client) of
         true ->
@@ -122,7 +136,7 @@ fields(basic) ->
             mk(enum([ns, us, ms, s, m, h]), #{
                 required => false, default => ms, desc => ?DESC("precision")
             })},
-        {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}
+        {pool_size, mk(pos_integer(), #{desc => ?DESC("pool_size")})}
     ];
 fields(influxdb_udp) ->
     fields(basic);

From d0e923590e09df6ac0b913a78340e8e7eeffffcf Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 15 Aug 2022 14:01:02 +0800
Subject: [PATCH 071/232] fix: write influxdb line with undefined value

---
 .../emqx_ee_connector/src/emqx_ee_connector_influxdb.erl   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 2c2de9a99..09a09aa44 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -502,7 +502,8 @@ maps_config_to_data(K, V, {Data, Res}) ->
     case {NK, NV} of
         {[undefined], _} ->
             {Data, Res};
-        {_, [undefined]} ->
+        %% undefined value in normal format [undefined] or int/uint format [undefined, <<"i">>]
+        {_, [undefined | _]} ->
             {Data, Res};
         _ ->
             {Data, Res#{NK => value_type(NV)}}
@@ -512,7 +513,9 @@ value_type([Int, <<"i">>]) when
     is_integer(Int)
 ->
     {int, Int};
-value_type([UInt, <<"u">>]) ->
+value_type([UInt, <<"u">>]) when
+    is_integer(UInt)
+->
     {uint, UInt};
 value_type([<<"t">>]) ->
     't';

From 665ef4142df02678a8177e31ef4ee24cc12101ea Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 15 Aug 2022 15:22:13 +0800
Subject: [PATCH 072/232] fix: unify the health check interval

---
 apps/emqx_resource/src/emqx_resource_manager.erl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 66d9e32b0..48353124c 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -56,7 +56,6 @@
 -record(data, {id, manager_id, group, mod, callback_mode, config, opts, status, state, error}).
 -type data() :: #data{}.
 
--define(SHORT_HEALTHCHECK_INTERVAL, 1000).
 -define(ETS_TABLE, ?MODULE).
 -define(WAIT_FOR_RESOURCE_DELAY, 100).
 -define(T_OPERATION, 5000).
@@ -484,7 +483,7 @@ handle_connecting_health_check(Data) ->
             (connected, UpdatedData) ->
                 {next_state, connected, UpdatedData};
             (connecting, UpdatedData) ->
-                Actions = [{state_timeout, ?SHORT_HEALTHCHECK_INTERVAL, health_check}],
+                Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}],
                 {keep_state, UpdatedData, Actions};
             (disconnected, UpdatedData) ->
                 {next_state, disconnected, UpdatedData}

From d1de262f31397cd4587e44111a1a97d169adf212 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 15 Aug 2022 15:23:15 +0800
Subject: [PATCH 073/232] fix: inc 'actions.failed' if bridge query failed

---
 .../include/emqx_resource_errors.hrl          | 20 +++++++++++++++++++
 .../src/emqx_resource_worker.erl              |  1 +
 .../src/emqx_rule_runtime.erl                 | 18 ++++++++++++++++-
 3 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 apps/emqx_resource/include/emqx_resource_errors.hrl

diff --git a/apps/emqx_resource/include/emqx_resource_errors.hrl b/apps/emqx_resource/include/emqx_resource_errors.hrl
new file mode 100644
index 000000000..b11ee3c1a
--- /dev/null
+++ b/apps/emqx_resource/include/emqx_resource_errors.hrl
@@ -0,0 +1,20 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-define(RESOURCE_ERROR(Reason, Msg),
+    {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}}
+).
+-define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index e940dcb69..f27f19bdf 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -21,6 +21,7 @@
 
 -include("emqx_resource.hrl").
 -include("emqx_resource_utils.hrl").
+-include("emqx_resource_errors.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index a0d1c464a..3729d9096 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -19,6 +19,7 @@
 -include("rule_engine.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("emqx_resource/include/emqx_resource_errors.hrl").
 
 -export([
     apply_rule/3,
@@ -322,7 +323,7 @@ handle_action(RuleId, ActId, Selected, Envs) ->
     ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.total'),
     try
         Result = do_handle_action(ActId, Selected, Envs),
-        ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success'),
+        inc_action_metrics(Result, RuleId),
         Result
     catch
         throw:out_of_service ->
@@ -501,3 +502,18 @@ ensure_list(_NotList) -> [].
 nested_put(Alias, Val, Columns0) ->
     Columns = handle_alias(Alias, Columns0),
     emqx_rule_maps:nested_put(Alias, Val, Columns).
+
+-define(IS_RES_DOWN(R), R == stopped; R == not_connected; R == not_found).
+inc_action_metrics(ok, RuleId) ->
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success');
+inc_action_metrics({ok, _}, RuleId) ->
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success');
+inc_action_metrics({resource_down, _}, RuleId) ->
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.out_of_service'),
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown');
+inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) ->
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.out_of_service'),
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown');
+inc_action_metrics(_, RuleId) ->
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'),
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown').

From 19d85d485bbe75941e43abe65cfbc648d5644897 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 15 Aug 2022 09:22:54 +0800
Subject: [PATCH 074/232] refactor(resource): add resource_opts level into
 config structure

---
 .../src/schema/emqx_bridge_webhook_schema.erl | 29 +++++++++++++------
 .../src/emqx_connector_mysql.erl              | 26 ++++++++++-------
 .../src/emqx_connector_utils.erl              | 19 ++++++++++++
 .../i18n/emqx_resource_schema_i18n.conf       | 21 +++++++-------
 apps/emqx_resource/include/emqx_resource.hrl  |  2 --
 apps/emqx_resource/src/emqx_resource.erl      | 16 +---------
 .../src/emqx_resource_worker.erl              |  7 +----
 .../src/schema/emqx_resource_schema.erl       | 21 +++++++++-----
 .../i18n/emqx_ee_bridge_mysql.conf            | 21 ++++++++++++++
 .../src/emqx_ee_bridge_influxdb.erl           |  2 +-
 .../src/emqx_ee_bridge_mysql.erl              | 10 ++++---
 11 files changed, 107 insertions(+), 67 deletions(-)
 create mode 100644 apps/emqx_connector/src/emqx_connector_utils.erl

diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
index d51c2edef..f5559f4c2 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
@@ -3,7 +3,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
--import(hoconsc, [mk/2, enum/1]).
+-import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -export([roots/0, fields/1, namespace/0, desc/1]).
 
@@ -23,7 +23,14 @@ fields("post") ->
 fields("put") ->
     fields("config");
 fields("get") ->
-    emqx_bridge_schema:metrics_status_fields() ++ fields("post").
+    emqx_bridge_schema:metrics_status_fields() ++ fields("post");
+fields("creation_opts") ->
+    lists:filter(
+        fun({K, _V}) ->
+            not lists:member(K, unsupported_opts())
+        end,
+        emqx_resource_schema:fields("creation_opts")
+    ).
 
 desc("config") ->
     ?DESC("desc_config");
@@ -117,13 +124,17 @@ request_config() ->
     ].
 
 webhook_creation_opts() ->
-    Opts = emqx_resource_schema:fields(creation_opts),
-    lists:filter(
-        fun({K, _V}) ->
-            not lists:member(K, unsupported_opts())
-        end,
-        Opts
-    ).
+    [
+        {resource_opts,
+            mk(
+                ref(?MODULE, "creation_opts"),
+                #{
+                    required => false,
+                    default => #{},
+                    desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
+                }
+            )}
+    ].
 
 unsupported_opts() ->
     [
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index cae07433a..a9b1cf08d 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -110,9 +110,14 @@ on_start(
         {auto_reconnect, reconn_interval(AutoReconn)},
         {pool_size, PoolSize}
     ],
+    SqlTmpl = emqx_map_lib:deep_get([egress, sql_template], Config, undefined),
+    SqlTmplParts = emqx_connector_utils:split_insert_sql(SqlTmpl),
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
     Prepares = parse_prepare_sql(Config),
-    State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares),
+    State0 = #{
+        poolname => PoolName, auto_reconnect => AutoReconn, sql_template_parts => SqlTmplParts
+    },
+    State = maps:merge(State0, Prepares),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
         ok -> {ok, init_prepare(State)};
         {error, Reason} -> {error, Reason}
@@ -136,18 +141,17 @@ on_query(
 ) ->
     LogMeta = #{connector => InstId, sql => SQLOrKey, state => State},
     ?TRACE("QUERY", "mysql_connector_received", LogMeta),
-    Worker = ecpool:get_client(PoolName),
-    {ok, Conn} = ecpool_worker:client(Worker),
-    MySqlFunction = mysql_function(TypeOrKey),
-    {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State),
-    Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey2, Data, Timeout]),
+    {ok, Conn} = ecpool_worker:client(ecpool:get_client(PoolName)),
+    MySqlFun = mysql_function(TypeOrKey),
+    {SQLOrKey2, SqlParams} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State),
+    Result = mysql:MySqlFun(Conn, SQLOrKey2, SqlParams, Timeout),
     case Result of
         {error, disconnected} ->
             ?SLOG(
                 error,
                 LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected}
             ),
-            %% kill the poll worker to trigger reconnection
+            %% kill the ecpool worker to trigger reconnection
             _ = exit(Conn, restart),
             Result;
         {error, not_prepared} ->
@@ -182,7 +186,7 @@ mysql_function(prepared_query) ->
     execute;
 %% for bridge
 mysql_function(_) ->
-    mysql_function(prepared_query).
+    execute.
 
 on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) ->
     case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of
@@ -328,10 +332,10 @@ proc_sql_params(query, SQLOrKey, Params, _State) ->
     {SQLOrKey, Params};
 proc_sql_params(prepared_query, SQLOrKey, Params, _State) ->
     {SQLOrKey, Params};
-proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) ->
-    case maps:get(TypeOrKey, ParamsTokens, undefined) of
+proc_sql_params(PreparedKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) ->
+    case maps:get(PreparedKey, ParamsTokens, undefined) of
         undefined ->
             {SQLOrData, Params};
         Tokens ->
-            {TypeOrKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
+            {PreparedKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
     end.
diff --git a/apps/emqx_connector/src/emqx_connector_utils.erl b/apps/emqx_connector/src/emqx_connector_utils.erl
new file mode 100644
index 000000000..94b12921d
--- /dev/null
+++ b/apps/emqx_connector/src/emqx_connector_utils.erl
@@ -0,0 +1,19 @@
+-module(emqx_connector_utils).
+
+-export([split_insert_sql/1]).
+
+%% SQL = <<"INSERT INTO \"abc\" (c1,c2,c3) VALUES (${1}, ${1}, ${1})">>
+split_insert_sql(SQL) ->
+    case re:split(SQL, "((?i)values)", [{return, binary}]) of
+        [Part1, _, Part3] ->
+            case string:trim(Part1, leading) of
+                <<"insert", _/binary>> = InsertSQL ->
+                    {ok, {InsertSQL, Part3}};
+                <<"INSERT", _/binary>> = InsertSQL ->
+                    {ok, {InsertSQL, Part3}};
+                _ ->
+                    {error, not_insert_sql}
+            end;
+        _ ->
+            {error, not_insert_sql}
+    end.
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index 3ec170ebf..5ff138b0b 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -1,4 +1,14 @@
 emqx_resource_schema {
+  resource_opts {
+    desc {
+      en: """Resource options."""
+      zh: """资源相关的选项。"""
+    }
+    label {
+      en: """Resource Options"""
+      zh: """资源选项"""
+    }
+  }
 
   health_check_interval {
     desc {
@@ -86,17 +96,6 @@ The auto restart interval after the resource is disconnected, in milliseconds.
     }
   }
 
-  resume_interval {
-    desc {
-      en: """Resume time interval when resource down."""
-      zh: """资源不可用时的重试时间"""
-    }
-    label {
-      en: """resume_interval"""
-      zh: """恢复时间"""
-    }
-  }
-
   async_inflight_window {
     desc {
       en: """Async query inflight window."""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 190c278ae..a21c3583b 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -66,7 +66,6 @@
     enable_queue => boolean(),
     queue_max_bytes => integer(),
     query_mode => async | sync | dynamic,
-    resume_interval => integer(),
     async_inflight_window => integer()
 }.
 -type query_result() ::
@@ -81,7 +80,6 @@
 -define(DEFAULT_INFLIGHT, 100).
 -define(HEALTHCHECK_INTERVAL, 15000).
 -define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>).
--define(RESUME_INTERVAL, 15000).
 -define(START_AFTER_CREATED, true).
 -define(START_TIMEOUT, 5000).
 -define(START_TIMEOUT_RAW, <<"5s">>).
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 60f0dd360..e3bcc423d 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -280,21 +280,7 @@ get_instance(ResId) ->
 
 -spec fetch_creation_opts(map()) -> creation_opts().
 fetch_creation_opts(Opts) ->
-    SupportedOpts = [
-        health_check_interval,
-        start_timeout,
-        start_after_created,
-        auto_restart_interval,
-        enable_batch,
-        batch_size,
-        batch_time,
-        enable_queue,
-        queue_max_bytes,
-        query_mode,
-        resume_interval,
-        async_inflight_window
-    ],
-    maps:with(SupportedOpts, Opts).
+    maps:get(resource_opts, Opts).
 
 -spec list_instances() -> [resource_id()].
 list_instances() ->
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index f27f19bdf..457f8c91a 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -59,11 +59,6 @@
 -define(REPLY(FROM, REQUEST, RESULT), {reply, FROM, REQUEST, RESULT}).
 -define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]).
 
--define(RESOURCE_ERROR(Reason, Msg),
-    {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}}
-).
--define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
-
 -type id() :: binary().
 -type query() :: {query, from(), request()}.
 -type request() :: term().
@@ -140,7 +135,7 @@ init({Id, Index, Opts}) ->
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
         queue => Queue,
-        resume_interval => maps:get(resume_interval, Opts, ?RESUME_INTERVAL),
+        resume_interval => maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL),
         acc => [],
         acc_left => BatchSize,
         tref => undefined
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index ccc31a707..446ee8ad1 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -30,14 +30,25 @@ namespace() -> "resource_schema".
 
 roots() -> [].
 
-fields('creation_opts') ->
+fields("resource_opts") ->
+    [
+        {resource_opts,
+            mk(
+                ref(?MODULE, "creation_opts"),
+                #{
+                    required => false,
+                    default => #{},
+                    desc => ?DESC(<<"resource_opts">>)
+                }
+            )}
+    ];
+fields("creation_opts") ->
     [
         {health_check_interval, fun health_check_interval/1},
         {start_after_created, fun start_after_created/1},
         {start_timeout, fun start_timeout/1},
         {auto_restart_interval, fun auto_restart_interval/1},
         {query_mode, fun query_mode/1},
-        {resume_interval, fun resume_interval/1},
         {async_inflight_window, fun async_inflight_window/1},
         {enable_batch, fun enable_batch/1},
         {batch_size, fun batch_size/1},
@@ -88,12 +99,6 @@ enable_queue(default) -> false;
 enable_queue(desc) -> ?DESC("enable_queue");
 enable_queue(_) -> undefined.
 
-resume_interval(type) -> emqx_schema:duration_ms();
-resume_interval(desc) -> ?DESC("resume_interval");
-resume_interval(default) -> ?RESUME_INTERVAL;
-resume_interval(required) -> false;
-resume_interval(_) -> undefined.
-
 async_inflight_window(type) -> pos_integer();
 async_inflight_window(desc) -> ?DESC("async_inflight_window");
 async_inflight_window(default) -> ?DEFAULT_INFLIGHT;
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
index 48fbd1007..98716ff7e 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
@@ -1,4 +1,25 @@
 emqx_ee_bridge_mysql {
+
+    local_topic {
+        desc {
+            en: """
+The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic
+matching the local_topic will be forwarded.</br>
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is
+configured, then both the data got from the rule and the MQTT messages that match local_topic
+will be forwarded.
+"""
+            zh: """
+发送到 'local_topic' 的消息都会转发到 MySQL。 </br>
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。
+"""
+        }
+        label {
+                en: "Local Topic"
+                zh: "本地 Topic"
+            }
+    }
+
     sql_template {
         desc {
             en: """SQL Template"""
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 6a1b2677f..dce315721 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -91,7 +91,7 @@ fields(basic) ->
         {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
         {write_syntax, fun write_syntax/1}
     ] ++
-        emqx_resource_schema:fields('creation_opts');
+        emqx_resource_schema:fields("resource_opts");
 fields("post_udp") ->
     method_fileds(post, influxdb_udp);
 fields("post_api_v1") ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index 5d143bf85..c9b611a52 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -43,14 +43,15 @@ values(get) ->
 values(post) ->
     #{
         type => mysql,
-        name => <<"mysql">>,
+        name => <<"foo">>,
+        local_topic => <<"local/topic/#">>,
         sql_template => ?DEFAULT_SQL,
         connector => #{
             server => <<"127.0.0.1:3306">>,
             database => <<"test">>,
             pool_size => 8,
             username => <<"root">>,
-            password => <<"public">>,
+            password => <<"">>,
             auto_reconnect => true
         },
         enable => true,
@@ -61,7 +62,7 @@ values(put) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
-namespace() -> "bridge".
+namespace() -> "bridge_mysql".
 
 roots() -> [].
 
@@ -69,6 +70,7 @@ fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
+        {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
         {sql_template,
             mk(
                 binary(),
@@ -82,7 +84,7 @@ fields("config") ->
                     desc => ?DESC("desc_connector")
                 }
             )}
-    ];
+    ] ++ emqx_resource_schema:fields("resource_opts");
 fields("post") ->
     [type_field(), name_field() | fields("config")];
 fields("put") ->

From de3a3259539148d031c8902bcbee41943f245cb8 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 16 Aug 2022 09:05:22 +0800
Subject: [PATCH 075/232] fix: revert the changes in connector mysql

---
 .../src/emqx_connector_mysql.erl              | 26 ++++++++-----------
 apps/emqx_resource/src/emqx_resource.erl      |  2 +-
 .../src/emqx_resource_manager.erl             |  5 ++++
 .../src/emqx_rule_engine.app.src              |  2 +-
 .../i18n/emqx_ee_bridge_mysql.conf            |  6 ++---
 5 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index a9b1cf08d..cae07433a 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -110,14 +110,9 @@ on_start(
         {auto_reconnect, reconn_interval(AutoReconn)},
         {pool_size, PoolSize}
     ],
-    SqlTmpl = emqx_map_lib:deep_get([egress, sql_template], Config, undefined),
-    SqlTmplParts = emqx_connector_utils:split_insert_sql(SqlTmpl),
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
     Prepares = parse_prepare_sql(Config),
-    State0 = #{
-        poolname => PoolName, auto_reconnect => AutoReconn, sql_template_parts => SqlTmplParts
-    },
-    State = maps:merge(State0, Prepares),
+    State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
         ok -> {ok, init_prepare(State)};
         {error, Reason} -> {error, Reason}
@@ -141,17 +136,18 @@ on_query(
 ) ->
     LogMeta = #{connector => InstId, sql => SQLOrKey, state => State},
     ?TRACE("QUERY", "mysql_connector_received", LogMeta),
-    {ok, Conn} = ecpool_worker:client(ecpool:get_client(PoolName)),
-    MySqlFun = mysql_function(TypeOrKey),
-    {SQLOrKey2, SqlParams} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State),
-    Result = mysql:MySqlFun(Conn, SQLOrKey2, SqlParams, Timeout),
+    Worker = ecpool:get_client(PoolName),
+    {ok, Conn} = ecpool_worker:client(Worker),
+    MySqlFunction = mysql_function(TypeOrKey),
+    {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State),
+    Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey2, Data, Timeout]),
     case Result of
         {error, disconnected} ->
             ?SLOG(
                 error,
                 LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected}
             ),
-            %% kill the ecpool worker to trigger reconnection
+            %% kill the poll worker to trigger reconnection
             _ = exit(Conn, restart),
             Result;
         {error, not_prepared} ->
@@ -186,7 +182,7 @@ mysql_function(prepared_query) ->
     execute;
 %% for bridge
 mysql_function(_) ->
-    execute.
+    mysql_function(prepared_query).
 
 on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) ->
     case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of
@@ -332,10 +328,10 @@ proc_sql_params(query, SQLOrKey, Params, _State) ->
     {SQLOrKey, Params};
 proc_sql_params(prepared_query, SQLOrKey, Params, _State) ->
     {SQLOrKey, Params};
-proc_sql_params(PreparedKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) ->
-    case maps:get(PreparedKey, ParamsTokens, undefined) of
+proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) ->
+    case maps:get(TypeOrKey, ParamsTokens, undefined) of
         undefined ->
             {SQLOrData, Params};
         Tokens ->
-            {PreparedKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
+            {TypeOrKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
     end.
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index a8fe297bf..0295292dd 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -298,7 +298,7 @@ get_instance(ResId) ->
 
 -spec fetch_creation_opts(map()) -> creation_opts().
 fetch_creation_opts(Opts) ->
-    maps:get(resource_opts, Opts).
+    maps:get(resource_opts, Opts, #{}).
 
 -spec list_instances() -> [resource_id()].
 list_instances() ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 48353124c..07abd4007 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -447,6 +447,11 @@ start_resource(Data, From) ->
             Actions = maybe_reply([{state_timeout, 0, health_check}], From, ok),
             {next_state, connecting, UpdatedData, Actions};
         {error, Reason} = Err ->
+            ?SLOG(error, #{
+                msg => start_resource_failed,
+                id => Data#data.id,
+                reason => Reason
+            }),
             _ = maybe_alarm(disconnected, Data#data.id),
             %% Keep track of the error reason why the connection did not work
             %% so that the Reason can be returned when the verification call is made.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 61c0f4ac1..28f90fdb9 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.1"},
+    {vsn, "5.0.2"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt]},
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
index 98716ff7e..bb908628c 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
@@ -2,15 +2,13 @@ emqx_ee_bridge_mysql {
 
     local_topic {
         desc {
-            en: """
-The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic
+            en: """The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic
 matching the local_topic will be forwarded.</br>
 NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is
 configured, then both the data got from the rule and the MQTT messages that match local_topic
 will be forwarded.
 """
-            zh: """
-发送到 'local_topic' 的消息都会转发到 MySQL。 </br>
+            zh: """发送到 'local_topic' 的消息都会转发到 MySQL。 </br>
 注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。
 """
         }

From 9e35032d78f569372ab26b37f7bee6a5720f7c8e Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 16 Aug 2022 10:09:02 +0800
Subject: [PATCH 076/232] fix: make resume_interval defaults to
 health_check_interval

---
 .../src/schema/emqx_bridge_webhook_schema.erl        |  2 ++
 .../i18n/emqx_resource_schema_i18n.conf              | 12 ++++++++++++
 apps/emqx_resource/include/emqx_resource.hrl         |  1 +
 apps/emqx_resource/src/emqx_resource_worker.erl      |  2 +-
 .../src/schema/emqx_resource_schema.erl              |  5 ++++-
 5 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
index 717acdfad..7c19e9e55 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
@@ -34,6 +34,8 @@ fields("creation_opts") ->
 
 desc("config") ->
     ?DESC("desc_config");
+desc("creation_opts") ->
+    ?DESC(emqx_resource_schema, "creation_opts");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for WebHook using `", string:to_upper(Method), "` method."];
 desc(_) ->
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index fab55badb..ce4c7e3b0 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -1,4 +1,5 @@
 emqx_resource_schema {
+
   resource_opts {
     desc {
       en: """Resource options."""
@@ -10,6 +11,17 @@ emqx_resource_schema {
     }
   }
 
+  creation_opts {
+    desc {
+      en: """Creation options."""
+      zh: """资源启动相关的选项。"""
+    }
+    label {
+      en: """Creation Options"""
+      zh: """资源启动选项"""
+    }
+  }
+
   health_check_interval {
     desc {
       en: """Health check interval, in milliseconds."""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 988950420..04b3f16ea 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -66,6 +66,7 @@
     enable_queue => boolean(),
     queue_max_bytes => integer(),
     query_mode => async | sync | dynamic,
+    resume_interval => integer(),
     async_inflight_window => integer()
 }.
 -type query_result() ::
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 457f8c91a..26b9706c9 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -135,7 +135,7 @@ init({Id, Index, Opts}) ->
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
         queue => Queue,
-        resume_interval => maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL),
+        resume_interval => maps:get(resume_interval, Opts, ?HEALTHCHECK_INTERVAL),
         acc => [],
         acc_left => BatchSize,
         tref => undefined
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index ed13f91e3..2272234f2 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -21,7 +21,7 @@
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
--export([namespace/0, roots/0, fields/1]).
+-export([namespace/0, roots/0, fields/1, desc/1]).
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
@@ -122,3 +122,6 @@ queue_max_bytes(desc) -> ?DESC("queue_max_bytes");
 queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW;
 queue_max_bytes(required) -> false;
 queue_max_bytes(_) -> undefined.
+
+desc("creation_opts") ->
+    ?DESC("creation_opts").

From 8d8afd1688722a877fcecf43448c93df8e63b753 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Fri, 19 Aug 2022 13:16:22 +0800
Subject: [PATCH 077/232] feat(bridge): add `on_batch_query` on
 emqx_connector_mysql

---
 .../src/emqx_connector_mysql.erl              | 154 ++++++++++++++----
 .../src/emqx_plugin_libs_rule.erl             |  41 ++++-
 .../src/emqx_ee_bridge_mysql.erl              |  16 +-
 3 files changed, 176 insertions(+), 35 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index cae07433a..bedd09267 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -28,6 +28,7 @@
     on_start/2,
     on_stop/2,
     on_query/3,
+    on_batch_query/3,
     on_get_status/2
 ]).
 
@@ -47,12 +48,15 @@
 
 -type prepares() :: #{atom() => binary()}.
 -type params_tokens() :: #{atom() => list()}.
+-type sqls() :: #{atom() => binary()}.
 -type state() ::
     #{
         poolname := atom(),
-        prepare_statement := prepares(),
         auto_reconnect := boolean(),
-        params_tokens := params_tokens()
+        prepare_statement := prepares(),
+        params_tokens := params_tokens(),
+        batch_inserts := sqls(),
+        batch_params_tokens := params_tokens()
     }.
 
 %%=====================================================================
@@ -134,48 +138,46 @@ on_query(
     {TypeOrKey, SQLOrKey, Params, Timeout},
     #{poolname := PoolName, prepare_statement := Prepares} = State
 ) ->
-    LogMeta = #{connector => InstId, sql => SQLOrKey, state => State},
-    ?TRACE("QUERY", "mysql_connector_received", LogMeta),
-    Worker = ecpool:get_client(PoolName),
-    {ok, Conn} = ecpool_worker:client(Worker),
     MySqlFunction = mysql_function(TypeOrKey),
     {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State),
-    Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey2, Data, Timeout]),
-    case Result of
-        {error, disconnected} ->
-            ?SLOG(
-                error,
-                LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected}
-            ),
-            %% kill the poll worker to trigger reconnection
-            _ = exit(Conn, restart),
-            Result;
+    case on_sql_query(InstId, MySqlFunction, SQLOrKey2, Data, Timeout, State) of
         {error, not_prepared} ->
-            ?SLOG(
-                warning,
-                LogMeta#{msg => "mysql_connector_prepare_query_failed", reason => not_prepared}
-            ),
             case prepare_sql(Prepares, PoolName) of
                 ok ->
                     %% not return result, next loop will try again
                     on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, State);
                 {error, Reason} ->
+                    LogMeta = #{connector => InstId, sql => SQLOrKey, state => State},
                     ?SLOG(
                         error,
                         LogMeta#{msg => "mysql_connector_do_prepare_failed", reason => Reason}
                     ),
                     {error, Reason}
             end;
-        {error, Reason} ->
-            ?SLOG(
-                error,
-                LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
-            ),
-            Result;
-        _ ->
+        Result ->
             Result
     end.
 
+on_batch_query(
+    InstId,
+    BatchReq,
+    #{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State
+) ->
+    case hd(BatchReq) of
+        {Key, _} ->
+            case maps:get(Key, Inserts, undefined) of
+                undefined ->
+                    {error, batch_select_not_implemented};
+                InsertSQL ->
+                    Tokens = maps:get(Key, ParamsTokens),
+                    on_batch_insert(InstId, BatchReq, InsertSQL, Tokens, State)
+            end;
+        Request ->
+            LogMeta = #{connector => InstId, first_request => Request, state => State},
+            ?SLOG(error, LogMeta#{msg => "invalid request"}),
+            {error, invald_request}
+    end.
+
 mysql_function(sql) ->
     query;
 mysql_function(prepared_query) ->
@@ -316,13 +318,44 @@ parse_prepare_sql(Config) ->
             Any ->
                 Any
         end,
-    parse_prepare_sql(maps:to_list(SQL), #{}, #{}).
+    parse_prepare_sql(maps:to_list(SQL), #{}, #{}, #{}, #{}).
 
-parse_prepare_sql([{Key, H} | T], SQL, Tokens) ->
+parse_prepare_sql([{Key, H} | _] = L, Prepares, Tokens, BatchInserts, BatchTks) ->
     {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H),
-    parse_prepare_sql(T, SQL#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens});
-parse_prepare_sql([], SQL, Tokens) ->
-    #{prepare_statement => SQL, params_tokens => Tokens}.
+    parse_batch_prepare_sql(
+        L, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}, BatchInserts, BatchTks
+    );
+parse_prepare_sql([], Prepares, Tokens, BatchInserts, BatchTks) ->
+    #{
+        prepare_statement => Prepares,
+        params_tokens => Tokens,
+        batch_inserts => BatchInserts,
+        batch_params_tokens => BatchTks
+    }.
+
+parse_batch_prepare_sql([{Key, H} | T], Prepares, Tokens, BatchInserts, BatchTks) ->
+    case emqx_plugin_libs_rule:detect_sql_type(H) of
+        {ok, select} ->
+            parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks);
+        {ok, insert} ->
+            case emqx_plugin_libs_rule:split_insert_sql(H) of
+                {ok, {InsertSQL, Params}} ->
+                    ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(Params),
+                    parse_prepare_sql(
+                        T,
+                        Prepares,
+                        Tokens,
+                        BatchInserts#{Key => InsertSQL},
+                        BatchTks#{Key => ParamsTks}
+                    );
+                {error, Reason} ->
+                    ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}),
+                    parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks)
+            end;
+        {error, Reason} ->
+            ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}),
+            parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks)
+    end.
 
 proc_sql_params(query, SQLOrKey, Params, _State) ->
     {SQLOrKey, Params};
@@ -335,3 +368,60 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens})
         Tokens ->
             {TypeOrKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
     end.
+
+on_batch_insert(InstId, BatchReqs, InsertPart, Tokens, State) ->
+    JoinFun = fun
+        ([Msg]) ->
+            emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg);
+        ([H | T]) ->
+            lists:foldl(
+                fun(Msg, Acc) ->
+                    Value = emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg),
+                    <<Acc/binary, ", ", Value/binary>>
+                end,
+                emqx_plugin_libs_rule:proc_sql_param_str(Tokens, H),
+                T
+            )
+    end,
+    {_, Msgs} = lists:unzip(BatchReqs),
+    JoinPart = JoinFun(Msgs),
+    SQL = <<InsertPart/binary, " values ", JoinPart/binary>>,
+    on_sql_query(InstId, query, SQL, [], default_timeout, State).
+
+on_sql_query(
+    InstId,
+    SQLFunc,
+    SQLOrKey,
+    Data,
+    Timeout,
+    #{poolname := PoolName} = State
+) ->
+    LogMeta = #{connector => InstId, sql => SQLOrKey, state => State},
+    ?TRACE("QUERY", "mysql_connector_received", LogMeta),
+    Worker = ecpool:get_client(PoolName),
+    {ok, Conn} = ecpool_worker:client(Worker),
+    Result = erlang:apply(mysql, SQLFunc, [Conn, SQLOrKey, Data, Timeout]),
+    case Result of
+        {error, disconnected} ->
+            ?SLOG(
+                error,
+                LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected}
+            ),
+            %% kill the poll worker to trigger reconnection
+            _ = exit(Conn, restart),
+            Result;
+        {error, not_prepared} = Error ->
+            ?SLOG(
+                warning,
+                LogMeta#{msg => "mysql_connector_prepare_query_failed", reason => not_prepared}
+            ),
+            Error;
+        {error, Reason} ->
+            ?SLOG(
+                error,
+                LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
+            ),
+            Result;
+        _ ->
+            Result
+    end.
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
index 03304c209..e94d62b53 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
@@ -29,7 +29,9 @@
     preproc_sql/2,
     proc_sql/2,
     proc_sql_param_str/2,
-    proc_cql_param_str/2
+    proc_cql_param_str/2,
+    split_insert_sql/1,
+    detect_sql_type/1
 ]).
 
 %% type converting
@@ -123,6 +125,43 @@ proc_sql_param_str(Tokens, Data) ->
 proc_cql_param_str(Tokens, Data) ->
     emqx_placeholder:proc_cql_param_str(Tokens, Data).
 
+%% SQL = <<"INSERT INTO \"abc\" (c1,c2,c3) VALUES (${1}, ${1}, ${1})">>
+-spec split_insert_sql(binary()) -> {ok, {InsertSQL, Params}} | {error, atom()} when
+    InsertSQL :: binary(),
+    Params :: binary().
+split_insert_sql(SQL) ->
+    case re:split(SQL, "((?i)values)", [{return, binary}]) of
+        [Part1, _, Part3] ->
+            case string:trim(Part1, leading) of
+                <<"insert", _/binary>> = InsertSQL ->
+                    {ok, {InsertSQL, Part3}};
+                <<"INSERT", _/binary>> = InsertSQL ->
+                    {ok, {InsertSQL, Part3}};
+                _ ->
+                    {error, not_insert_sql}
+            end;
+        _ ->
+            {error, not_insert_sql}
+    end.
+
+-spec detect_sql_type(binary()) -> {ok, Type} | {error, atom()} when
+    Type :: insert | select.
+detect_sql_type(SQL) ->
+    case re:run(SQL, "^\\s*([a-zA-Z]+)", [{capture, all_but_first, list}]) of
+        {match, [First]} ->
+            Types = [select, insert],
+            PropTypes = [{erlang:atom_to_list(Type), Type} || Type <- Types],
+            LowFirst = string:lowercase(First),
+            case proplists:lookup(LowFirst, PropTypes) of
+                {LowFirst, Type} ->
+                    {ok, Type};
+                _ ->
+                    {error, invalid_sql}
+            end;
+        _ ->
+            {error, invalid_sql}
+    end.
+
 unsafe_atom_key(Key) when is_atom(Key) ->
     Key;
 unsafe_atom_key(Key) when is_binary(Key) ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index c9b611a52..5f6db77e8 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -6,6 +6,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include("emqx_ee_bridge.hrl").
+-include_lib("emqx_resource/include/emqx_resource.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
@@ -44,7 +45,6 @@ values(post) ->
     #{
         type => mysql,
         name => <<"foo">>,
-        local_topic => <<"local/topic/#">>,
         sql_template => ?DEFAULT_SQL,
         connector => #{
             server => <<"127.0.0.1:3306">>,
@@ -54,6 +54,19 @@ values(post) ->
             password => <<"">>,
             auto_reconnect => true
         },
+        resource_opts => #{
+            health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
+            start_after_created => ?START_AFTER_CREATED,
+            start_timeout => ?START_TIMEOUT_RAW,
+            auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
+            query_mode => sync,
+            async_inflight_window => ?DEFAULT_INFLIGHT,
+            enable_batch => false,
+            batch_size => ?DEFAULT_BATCH_SIZE,
+            batch_time => ?DEFAULT_BATCH_TIME,
+            enable_queue => false,
+            max_queue_bytes => ?DEFAULT_QUEUE_SIZE
+        },
         enable => true,
         direction => egress
     };
@@ -70,7 +83,6 @@ fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
         {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
-        {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
         {sql_template,
             mk(
                 binary(),

From dce47aac1773e685569316151ebc4c1293c0130d Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Fri, 19 Aug 2022 15:31:37 +0800
Subject: [PATCH 078/232] fix(bridge): don't export internal fields

---
 .../src/emqx_ee_bridge_mysql.erl              | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index 5f6db77e8..c6ebe1fc5 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -56,11 +56,7 @@ values(post) ->
         },
         resource_opts => #{
             health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
-            start_after_created => ?START_AFTER_CREATED,
-            start_timeout => ?START_TIMEOUT_RAW,
             auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
-            query_mode => sync,
-            async_inflight_window => ?DEFAULT_INFLIGHT,
             enable_batch => false,
             batch_size => ?DEFAULT_BATCH_SIZE,
             batch_time => ?DEFAULT_BATCH_TIME,
@@ -95,8 +91,27 @@ fields("config") ->
                     required => true,
                     desc => ?DESC("desc_connector")
                 }
+            )},
+        {resource_opts,
+            mk(
+                ref(?MODULE, "creation_opts"),
+                #{
+                    required => false,
+                    default => #{},
+                    desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
+                }
             )}
-    ] ++ emqx_resource_schema:fields("resource_opts");
+    ];
+fields("creation_opts") ->
+    Opts = emqx_resource_schema:fields("creation_opts"),
+    lists:filter(
+        fun({Field, _}) ->
+            not lists:member(Field, [
+                start_after_created, start_timeout, query_mode, async_inflight_window
+            ])
+        end,
+        Opts
+    );
 fields("post") ->
     [type_field(), name_field() | fields("config")];
 fields("put") ->

From 06363e63d9caf61abacae9b162e97fcbfc314308 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 18 Aug 2022 16:00:04 +0800
Subject: [PATCH 079/232] fix(influxdb): connector use a fallbacke `pool_size`
 for influxdb client

---
 .../src/emqx_connector_schema_lib.erl         |  2 ++
 .../i18n/emqx_resource_schema_i18n.conf       | 11 +++++++++++
 apps/emqx_resource/include/emqx_resource.hrl  | 19 +++++++++++--------
 .../src/emqx_resource_worker_sup.erl          | 12 ++++++------
 .../src/schema/emqx_resource_schema.erl       |  7 +++++++
 .../i18n/emqx_ee_connector_influxdb.conf      | 10 ----------
 .../src/emqx_ee_connector_influxdb.erl        | 13 ++++---------
 7 files changed, 41 insertions(+), 33 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
index dd85566ed..3bd29a9c1 100644
--- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
@@ -68,6 +68,8 @@ ssl_fields() ->
 relational_db_fields() ->
     [
         {database, fun database/1},
+        %% TODO: The `pool_size` for drivers will be deprecated. Ues `worker_pool_size` for emqx_resource
+        %% See emqx_resource.hrl
         {pool_size, fun pool_size/1},
         {username, fun username/1},
         {password, fun password/1},
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index ce4c7e3b0..43a32288d 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -22,6 +22,17 @@ emqx_resource_schema {
     }
   }
 
+  worker_pool_size {
+    desc {
+      en: """Resource worker pool size."""
+      zh: """资源连接池大小。"""
+    }
+    label {
+      en: """Worker Pool Size"""
+      zh: """资源连接池大小"""
+    }
+  }
+
   health_check_interval {
     desc {
       en: """Health check interval, in milliseconds."""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 04b3f16ea..4bbb4beb6 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -49,25 +49,26 @@
     %% use auto_restart_interval instead
     auto_retry_interval => integer(),
     %%======================================= Deprecated Opts End
-    health_check_interval => integer(),
+    worker_pool_size => pos_integer(),
+    health_check_interval => pos_integer(),
     %% We can choose to block the return of emqx_resource:start until
     %% the resource connected, wait max to `start_timeout` ms.
-    start_timeout => integer(),
+    start_timeout => pos_integer(),
     %% If `start_after_created` is set to true, the resource is started right
     %% after it is created. But note that a `started` resource is not guaranteed
     %% to be `connected`.
     start_after_created => boolean(),
     %% If the resource disconnected, we can set to retry starting the resource
     %% periodically.
-    auto_restart_interval => integer(),
+    auto_restart_interval => pos_integer(),
     enable_batch => boolean(),
-    batch_size => integer(),
-    batch_time => integer(),
+    batch_size => pos_integer(),
+    batch_time => pos_integer(),
     enable_queue => boolean(),
-    queue_max_bytes => integer(),
+    queue_max_bytes => pos_integer(),
     query_mode => async | sync | dynamic,
-    resume_interval => integer(),
-    async_inflight_window => integer()
+    resume_interval => pos_integer(),
+    async_inflight_window => pos_integer()
 }.
 -type query_result() ::
     ok
@@ -75,6 +76,8 @@
     | {error, term()}
     | {resource_down, term()}.
 
+-define(WORKER_POOL_SIZE, 16).
+
 -define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
 -define(DEFAULT_QUEUE_SIZE_RAW, <<"1GB">>).
 
diff --git a/apps/emqx_resource/src/emqx_resource_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_worker_sup.erl
index a2b3a1ba5..5305eddaf 100644
--- a/apps/emqx_resource/src/emqx_resource_worker_sup.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker_sup.erl
@@ -53,23 +53,23 @@ init([]) ->
     {ok, {SupFlags, ChildSpecs}}.
 
 start_workers(ResId, Opts) ->
-    PoolSize = pool_size(Opts),
-    _ = ensure_worker_pool(ResId, hash, [{size, PoolSize}]),
+    WorkerPoolSize = worker_pool_size(Opts),
+    _ = ensure_worker_pool(ResId, hash, [{size, WorkerPoolSize}]),
     lists:foreach(
         fun(Idx) ->
             _ = ensure_worker_added(ResId, Idx),
             ok = ensure_worker_started(ResId, Idx, Opts)
         end,
-        lists:seq(1, PoolSize)
+        lists:seq(1, WorkerPoolSize)
     ).
 
 stop_workers(ResId, Opts) ->
-    PoolSize = pool_size(Opts),
+    WorkerPoolSize = worker_pool_size(Opts),
     lists:foreach(
         fun(Idx) ->
             ensure_worker_removed(ResId, Idx)
         end,
-        lists:seq(1, PoolSize)
+        lists:seq(1, WorkerPoolSize)
     ),
     ensure_worker_pool_removed(ResId),
     ok.
@@ -77,7 +77,7 @@ stop_workers(ResId, Opts) ->
 %%%=============================================================================
 %%% Internal
 %%%=============================================================================
-pool_size(Opts) ->
+worker_pool_size(Opts) ->
     maps:get(worker_pool_size, Opts, erlang:system_info(schedulers_online)).
 
 ensure_worker_pool(ResId, Type, Opts) ->
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index 2272234f2..77d9c3659 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -44,6 +44,7 @@ fields("resource_opts") ->
     ];
 fields("creation_opts") ->
     [
+        {worker_pool_size, fun worker_pool_size/1},
         {health_check_interval, fun health_check_interval/1},
         {start_after_created, fun start_after_created/1},
         {start_timeout, fun start_timeout/1},
@@ -57,6 +58,12 @@ fields("creation_opts") ->
         {max_queue_bytes, fun queue_max_bytes/1}
     ].
 
+worker_pool_size(type) -> pos_integer();
+worker_pool_size(desc) -> ?DESC("worker_pool_size");
+worker_pool_size(default) -> ?WORKER_POOL_SIZE;
+worker_pool_size(required) -> false;
+worker_pool_size(_) -> undefined.
+
 health_check_interval(type) -> emqx_schema:duration_ms();
 health_check_interval(desc) -> ?DESC("health_check_interval");
 health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW;
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
index 4d2dc168c..ff2266de5 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
@@ -150,15 +150,5 @@ emqx_ee_connector_influxdb {
                 zh: """时间精度"""
             }
     }
-    pool_size {
-        desc {
-                en: """InfluxDB Pool Size. Default value is CPU threads."""
-                zh: """InfluxDB 连接池大小，默认为 CPU 线程数。"""
-            }
-        label {
-                en: """InfluxDB Pool Size"""
-                zh: """InfluxDB 连接池大小"""
-            }
-    }
 
 }
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 09a09aa44..5ec96bf2c 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -135,8 +135,7 @@ fields(basic) ->
         {precision,
             mk(enum([ns, us, ms, s, m, h]), #{
                 required => false, default => ms, desc => ?DESC("precision")
-            })},
-        {pool_size, mk(pos_integer(), #{desc => ?DESC("pool_size")})}
+            })}
     ];
 fields(influxdb_udp) ->
     fields(basic);
@@ -190,15 +189,13 @@ values(udp, put) ->
     #{
         host => <<"127.0.0.1">>,
         port => 8089,
-        precision => ms,
-        pool_size => 8
+        precision => ms
     };
 values(api_v1, put) ->
     #{
         host => <<"127.0.0.1">>,
         port => 8086,
         precision => ms,
-        pool_size => 8,
         database => <<"my_db">>,
         username => <<"my_user">>,
         password => <<"my_password">>,
@@ -209,7 +206,6 @@ values(api_v2, put) ->
         host => <<"127.0.0.1">>,
         port => 8086,
         precision => ms,
-        pool_size => 8,
         bucket => <<"my_bucket">>,
         org => <<"my_org">>,
         token => <<"my_token">>,
@@ -302,14 +298,13 @@ client_config(
     InstId,
     Config = #{
         host := Host,
-        port := Port,
-        pool_size := PoolSize
+        port := Port
     }
 ) ->
     [
         {host, binary_to_list(Host)},
         {port, Port},
-        {pool_size, PoolSize},
+        {pool_size, erlang:system_info(schedulers)},
         {pool, binary_to_atom(InstId, utf8)},
         {precision, atom_to_binary(maps:get(precision, Config, ms), utf8)}
     ] ++ protocol_config(Config).

From 2648362c6216c8e5c2408edf098822576d1cf481 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 19 Aug 2022 15:53:22 +0800
Subject: [PATCH 080/232] fix(bridge): password for bridge/db format as
 `password` for dashboard

---
 apps/emqx/src/emqx_schema.erl                               | 1 +
 apps/emqx_connector/src/emqx_connector_schema_lib.erl       | 1 +
 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl | 1 +
 lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 4 ++--
 4 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 574305a4f..a76346bad 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -1902,6 +1902,7 @@ common_ssl_opts_schema(Defaults) ->
                     sensitive => true,
                     required => false,
                     example => <<"">>,
+                    format => <<"password">>,
                     desc => ?DESC(common_ssl_opts_schema_password)
                 }
             )},
diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
index 3bd29a9c1..f37640538 100644
--- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
@@ -104,6 +104,7 @@ username(_) -> undefined.
 password(type) -> binary();
 password(desc) -> ?DESC("password");
 password(required) -> false;
+password(format) -> <<"format">>;
 password(_) -> undefined.
 
 auto_reconnect(type) -> boolean();
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index 8b6994dd4..4d35583a3 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -96,6 +96,7 @@ fields("connector") ->
                 binary(),
                 #{
                     default => "emqx",
+                    format => <<"password">>,
                     desc => ?DESC("password")
                 }
             )},
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 5ec96bf2c..d2725c797 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -142,8 +142,8 @@ fields(influxdb_udp) ->
 fields(influxdb_api_v1) ->
     [
         {database, mk(binary(), #{required => true, desc => ?DESC("database")})},
-        {username, mk(binary(), #{required => true, desc => ?DESC("username")})},
-        {password, mk(binary(), #{required => true, desc => ?DESC("password")})}
+        {username, mk(binary(), #{desc => ?DESC("username")})},
+        {password, mk(binary(), #{desc => ?DESC("password"), format => <<"password">>})}
     ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic);
 fields(influxdb_api_v2) ->
     [

From 7c4ea38c06a7d44b07d786467e31ab6bcc64cfae Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 22 Aug 2022 02:20:54 +0800
Subject: [PATCH 081/232] fix(resource): make some resource opts internal

Resource options `start_after_created` and `start_timeout` are internal opts.
Not provided to users anymore.
---
 .../i18n/emqx_resource_schema_i18n.conf            |  4 ++--
 apps/emqx_resource/include/emqx_resource.hrl       |  6 ------
 apps/emqx_resource/src/emqx_resource_manager.erl   |  5 +++++
 .../src/schema/emqx_resource_schema.erl            | 14 --------------
 4 files changed, 7 insertions(+), 22 deletions(-)

diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index 43a32288d..0d53b813e 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -145,8 +145,8 @@ emqx_resource_schema {
 
   queue_max_bytes {
     desc {
-      en: """Maximum queue storage size in bytes."""
-      zh: """消息队列的最大长度，以字节计。"""
+      en: """Maximum queue storage."""
+      zh: """消息队列的最大长度。"""
     }
     label {
       en: """Queue max bytes"""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 4bbb4beb6..bb8cb02bf 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -95,12 +95,6 @@
 -define(HEALTHCHECK_INTERVAL, 15000).
 -define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>).
 
--define(START_AFTER_CREATED, true).
-
-%% milliseconds
--define(START_TIMEOUT, 5000).
--define(START_TIMEOUT_RAW, <<"5s">>).
-
 %% milliseconds
 -define(AUTO_RESTART_INTERVAL, 60000).
 -define(AUTO_RESTART_INTERVAL_RAW, <<"60s">>).
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 07abd4007..382cf29d9 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -114,6 +114,11 @@ create_and_return_data(MgrId, ResId, Group, ResourceType, Config, Opts) ->
     {ok, _Group, Data} = lookup(ResId),
     {ok, Data}.
 
+%% internal configs
+-define(START_AFTER_CREATED, true).
+%% in milliseconds
+-define(START_TIMEOUT, 5000).
+
 %% @doc Create a resource_manager and wait until it is running
 create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
     % The state machine will make the actual call to the callback/resource module after init
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index 77d9c3659..fe8564a41 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -46,8 +46,6 @@ fields("creation_opts") ->
     [
         {worker_pool_size, fun worker_pool_size/1},
         {health_check_interval, fun health_check_interval/1},
-        {start_after_created, fun start_after_created/1},
-        {start_timeout, fun start_timeout/1},
         {auto_restart_interval, fun auto_restart_interval/1},
         {query_mode, fun query_mode/1},
         {async_inflight_window, fun async_inflight_window/1},
@@ -70,18 +68,6 @@ health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW;
 health_check_interval(required) -> false;
 health_check_interval(_) -> undefined.
 
-start_after_created(type) -> boolean();
-start_after_created(required) -> false;
-start_after_created(default) -> ?START_AFTER_CREATED;
-start_after_created(desc) -> ?DESC("start_after_created");
-start_after_created(_) -> undefined.
-
-start_timeout(type) -> emqx_schema:duration_ms();
-start_timeout(desc) -> ?DESC("start_timeout");
-start_timeout(default) -> ?START_TIMEOUT_RAW;
-start_timeout(required) -> false;
-start_timeout(_) -> undefined.
-
 auto_restart_interval(type) -> hoconsc:union([infinity, emqx_schema:duration_ms()]);
 auto_restart_interval(desc) -> ?DESC("auto_restart_interval");
 auto_restart_interval(default) -> ?AUTO_RESTART_INTERVAL_RAW;

From 62ecf6f545652bcff3914852f22cac5f66e3aeb0 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 22 Aug 2022 02:34:25 +0800
Subject: [PATCH 082/232] fix(resource): keep `auto_retry` in `disconnected`
 state

Automatic retries should be maintained even in `disconnected` state without any state transition.
---
 .../src/emqx_resource_manager.erl             | 40 +++++++++++++++----
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 382cf29d9..226f9e927 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -384,9 +384,33 @@ handle_event(EventType, EventData, State, Data) ->
 %%------------------------------------------------------------------------------
 insert_cache(ResId, Group, Data = #data{manager_id = MgrId}) ->
     case get_owner(ResId) of
-        not_found -> ets:insert(?ETS_TABLE, {ResId, Group, Data});
-        MgrId -> ets:insert(?ETS_TABLE, {ResId, Group, Data});
-        _ -> self() ! quit
+        not_found ->
+            ?SLOG(
+                debug,
+                #{
+                    msg => resource_owner_not_found,
+                    resource_id => ResId,
+                    action => auto_insert_cache
+                }
+            ),
+            ets:insert(?ETS_TABLE, {ResId, Group, Data});
+        MgrId ->
+            ?SLOG(
+                debug,
+                #{
+                    msg => resource_owner_matched,
+                    resource_id => ResId,
+                    action => reinsert_cache
+                }
+            ),
+            ets:insert(?ETS_TABLE, {ResId, Group, Data});
+        _ ->
+            ?SLOG(error, #{
+                msg => get_resource_owner_failed,
+                resource_id => ResId,
+                action => quit_rusource
+            }),
+            self() ! quit
     end.
 
 read_cache(ResId) ->
@@ -425,12 +449,14 @@ get_owner(ResId) ->
     end.
 
 handle_disconnected_state_enter(Data) ->
+    {next_state, disconnected, Data, retry_actions(Data)}.
+
+retry_actions(Data) ->
     case maps:get(auto_restart_interval, Data#data.opts, ?AUTO_RESTART_INTERVAL) of
         undefined ->
-            {next_state, disconnected, Data};
+            [];
         RetryInterval ->
-            Actions = [{state_timeout, RetryInterval, auto_retry}],
-            {next_state, disconnected, Data, Actions}
+            [{state_timeout, RetryInterval, auto_retry}]
     end.
 
 handle_remove_event(From, ClearMetrics, Data) ->
@@ -461,7 +487,7 @@ start_resource(Data, From) ->
             %% Keep track of the error reason why the connection did not work
             %% so that the Reason can be returned when the verification call is made.
             UpdatedData = Data#data{error = Reason},
-            Actions = maybe_reply([], From, Err),
+            Actions = maybe_reply(retry_actions(UpdatedData), From, Err),
             {next_state, disconnected, UpdatedData, Actions}
     end.
 

From f0c2b53868f06e52d9ec6e27da687cd42823dcfe Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 22 Aug 2022 10:31:26 +0800
Subject: [PATCH 083/232] fix(bpapi): make bpapi static_checks happy

---
 .../src/emqx_connector_schema_lib.erl          |  2 +-
 apps/emqx_resource/include/emqx_resource.hrl   |  3 ++-
 .../src/emqx_resource_manager.erl              | 18 +-----------------
 3 files changed, 4 insertions(+), 19 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
index f37640538..53643c9f9 100644
--- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl
@@ -104,7 +104,7 @@ username(_) -> undefined.
 password(type) -> binary();
 password(desc) -> ?DESC("password");
 password(required) -> false;
-password(format) -> <<"format">>;
+password(format) -> <<"password">>;
 password(_) -> undefined.
 
 auto_reconnect(type) -> boolean();
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index bb8cb02bf..3f2cac46b 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -50,7 +50,8 @@
     auto_retry_interval => integer(),
     %%======================================= Deprecated Opts End
     worker_pool_size => pos_integer(),
-    health_check_interval => pos_integer(),
+    %% use `integer()` compatibility to release 5.0.0 bpapi
+    health_check_interval => integer(),
     %% We can choose to block the return of emqx_resource:start until
     %% the resource connected, wait max to `start_timeout` ms.
     start_timeout => pos_integer(),
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 226f9e927..2f6964380 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -385,30 +385,14 @@ handle_event(EventType, EventData, State, Data) ->
 insert_cache(ResId, Group, Data = #data{manager_id = MgrId}) ->
     case get_owner(ResId) of
         not_found ->
-            ?SLOG(
-                debug,
-                #{
-                    msg => resource_owner_not_found,
-                    resource_id => ResId,
-                    action => auto_insert_cache
-                }
-            ),
             ets:insert(?ETS_TABLE, {ResId, Group, Data});
         MgrId ->
-            ?SLOG(
-                debug,
-                #{
-                    msg => resource_owner_matched,
-                    resource_id => ResId,
-                    action => reinsert_cache
-                }
-            ),
             ets:insert(?ETS_TABLE, {ResId, Group, Data});
         _ ->
             ?SLOG(error, #{
                 msg => get_resource_owner_failed,
                 resource_id => ResId,
-                action => quit_rusource
+                action => quit_resource
             }),
             self() ! quit
     end.

From 2471580c476010673cf978e6bb2c5fdb1bb646cc Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Mon, 22 Aug 2022 15:58:11 +0800
Subject: [PATCH 084/232] fix(bridge): make spellcheck happy

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index c6ebe1fc5..e63052d50 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -127,6 +127,8 @@ desc(connector) ->
     ?DESC("desc_connector");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for MySQL using `", string:to_upper(Method), "` method."];
+desc("creation_opts" = Name) ->
+    emqx_resource_schema:desc(Name);
 desc(_) ->
     undefined.
 

From aea8c77b49b1f5b2cac377d54d1df543f1185d1e Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 22 Aug 2022 12:59:51 +0800
Subject: [PATCH 085/232] refactor: new config structure for mqtt bridge

---
 .../i18n/emqx_bridge_mqtt_schema.conf         |  12 -
 apps/emqx_bridge/i18n/emqx_bridge_schema.conf |  20 +-
 .../i18n/emqx_bridge_webhook_schema.conf      |  11 -
 apps/emqx_bridge/src/emqx_bridge.erl          |  51 ++-
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 153 ++++----
 apps/emqx_bridge/src/emqx_bridge_app.erl      |   1 -
 apps/emqx_bridge/src/emqx_bridge_resource.erl |  93 ++---
 .../src/schema/emqx_bridge_mqtt_schema.erl    |  44 +--
 .../src/schema/emqx_bridge_schema.erl         |  86 ++---
 .../src/schema/emqx_bridge_webhook_schema.erl |   8 -
 apps/emqx_conf/src/emqx_conf.erl              |   6 -
 apps/emqx_conf/src/emqx_conf_schema.erl       |   1 -
 .../i18n/emqx_connector_mqtt.conf             |   1 -
 .../i18n/emqx_connector_mqtt_schema.conf      | 210 +++++------
 .../i18n/emqx_connector_schema.conf           |  31 --
 apps/emqx_connector/src/emqx_connector.erl    | 165 ---------
 .../emqx_connector/src/emqx_connector_api.erl | 339 ------------------
 .../emqx_connector/src/emqx_connector_app.erl |   5 -
 .../src/emqx_connector_mqtt.erl               |   4 +-
 .../src/emqx_connector_schema.erl             |  95 -----
 .../src/mqtt/emqx_connector_mqtt_mod.erl      |  12 +-
 .../src/mqtt/emqx_connector_mqtt_msg.erl      |  34 +-
 .../src/mqtt/emqx_connector_mqtt_schema.erl   | 204 +++++------
 .../src/mqtt/emqx_connector_mqtt_worker.erl   |  32 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |   4 +-
 .../src/emqx_ee_connector.erl                 |  57 ---
 26 files changed, 378 insertions(+), 1301 deletions(-)
 delete mode 100644 apps/emqx_connector/i18n/emqx_connector_schema.conf
 delete mode 100644 apps/emqx_connector/src/emqx_connector.erl
 delete mode 100644 apps/emqx_connector/src/emqx_connector_api.erl
 delete mode 100644 apps/emqx_connector/src/emqx_connector_schema.erl
 delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl

diff --git a/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf
index c0f549db3..ab8c97ce7 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf
@@ -1,16 +1,4 @@
 emqx_bridge_mqtt_schema {
-
-    desc_rec {
-                   desc {
-                         en: """Configuration for MQTT bridge."""
-                         zh: """MQTT Bridge 配置"""
-                        }
-                   label: {
-                           en: "MQTT Bridge Configuration"
-                           zh: "MQTT Bridge 配置"
-                          }
-                  }
-
     desc_type {
                    desc {
                          en: """The bridge type."""
diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
index b1a8e56f9..704fd7bd7 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
@@ -11,24 +11,6 @@ emqx_bridge_schema {
                           }
                   }
 
-    desc_connector {
-                   desc {
-                         en: """
-The ID or the configs of the connector to be used for this bridge. Connector IDs must be of format:
-<code>{type}:{name}</code>.</br>
-In config files, you can find the corresponding config entry for a connector by such path:
-'connectors.{type}.{name}'.</br>
-"""
-                         zh: """
-Bridge 使用的 Connector 的 ID 或者配置。Connector ID 的格式必须为：<code>{type}:{name}</code>.</br>
-在配置文件中，您可以通过以下路径找到 Connector 的相应配置条目：'connector.{type}.{name}'。</br>"""
-                        }
-                   label: {
-                           en: "Connector ID"
-                           zh: "Connector ID"
-                          }
-                  }
-
     desc_metrics {
                    desc {
                          en: """The metrics of the bridge"""
@@ -85,7 +67,7 @@ Bridge 使用的 Connector 的 ID 或者配置。Connector ID 的格式必须为
                   }
 
 
-    bridges_name {
+    bridges_mqtt {
                    desc {
                          en: """MQTT bridges to/from another MQTT broker"""
                          zh: """桥接到另一个 MQTT Broker 的 MQTT Bridge"""
diff --git a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf
index fcc817bef..9e89b5f0c 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf
@@ -11,17 +11,6 @@ emqx_bridge_webhook_schema {
                           }
                   }
 
-    config_direction {
-                   desc {
-                         en: """The direction of this bridge, MUST be 'egress'"""
-                         zh: """Bridge 的方向， 必须是 egress"""
-                        }
-                   label: {
-                           en: "Bridge Direction"
-                           zh: "Bridge 方向"
-                          }
-                  }
-
     config_url {
                    desc {
                          en: """
diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index 354c4faee..109b1df86 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -37,8 +37,7 @@
     create/3,
     disable_enable/3,
     remove/2,
-    list/0,
-    list_bridges_by_connector/1
+    list/0
 ]).
 
 -export([send_message/2]).
@@ -48,6 +47,8 @@
 %% exported for `emqx_telemetry'
 -export([get_basic_usage_info/0]).
 
+-define(EGRESS_DIR_BRIDGES(T), T == webhook; T == mysql).
+
 load() ->
     Bridges = emqx:get_config([bridges], #{}),
     lists:foreach(
@@ -93,10 +94,10 @@ load_hook() ->
 
 load_hook(Bridges) ->
     lists:foreach(
-        fun({_Type, Bridge}) ->
+        fun({Type, Bridge}) ->
             lists:foreach(
                 fun({_Name, BridgeConf}) ->
-                    do_load_hook(BridgeConf)
+                    do_load_hook(Type, BridgeConf)
                 end,
                 maps:to_list(Bridge)
             )
@@ -104,12 +105,13 @@ load_hook(Bridges) ->
         maps:to_list(Bridges)
     ).
 
-do_load_hook(#{local_topic := _} = Conf) ->
-    case maps:get(direction, Conf, egress) of
-        egress -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_BRIDGE);
-        ingress -> ok
-    end;
-do_load_hook(_Conf) ->
+do_load_hook(Type, #{local_topic := _}) when ?EGRESS_DIR_BRIDGES(Type) ->
+    emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_BRIDGE);
+do_load_hook(mqtt, #{egress := #{local := #{topic := _}}}) ->
+    emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_BRIDGE);
+do_load_hook(kafka, #{producer := #{mqtt := #{topic := _}}}) ->
+    emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_BRIDGE);
+do_load_hook(_Type, _Conf) ->
     ok.
 
 unload_hook() ->
@@ -197,13 +199,6 @@ list() ->
         maps:to_list(emqx:get_raw_config([bridges], #{}))
     ).
 
-list_bridges_by_connector(ConnectorId) ->
-    [
-        B
-     || B = #{raw_config := #{<<"connector">> := Id}} <- list(),
-        ConnectorId =:= Id
-    ].
-
 lookup(Id) ->
     {Type, Name} = emqx_bridge_resource:parse_bridge_id(Id),
     lookup(Type, Name).
@@ -303,13 +298,8 @@ get_matched_bridges(Topic) ->
     maps:fold(
         fun(BType, Conf, Acc0) ->
             maps:fold(
-                fun
-                    %% Confs for MQTT, Kafka bridges have the `direction` flag
-                    (_BName, #{direction := ingress}, Acc1) ->
-                        Acc1;
-                    (BName, #{direction := egress} = Egress, Acc1) ->
-                        %% WebHook, MySQL bridges only have egress direction
-                        get_matched_bridge_id(Egress, Topic, BType, BName, Acc1)
+                fun(BName, BConf, Acc1) ->
+                    get_matched_bridge_id(BType, BConf, Topic, BName, Acc1)
                 end,
                 Acc0,
                 Conf
@@ -319,9 +309,18 @@ get_matched_bridges(Topic) ->
         Bridges
     ).
 
-get_matched_bridge_id(#{enable := false}, _Topic, _BType, _BName, Acc) ->
+get_matched_bridge_id(_BType, #{enable := false}, _Topic, _BName, Acc) ->
     Acc;
-get_matched_bridge_id(#{local_topic := Filter}, Topic, BType, BName, Acc) ->
+get_matched_bridge_id(BType, #{local_topic := Filter}, Topic, BName, Acc) when
+    ?EGRESS_DIR_BRIDGES(BType)
+->
+    do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc);
+get_matched_bridge_id(mqtt, #{egress := #{local := #{topic := Filter}}}, Topic, BName, Acc) ->
+    do_get_matched_bridge_id(Topic, Filter, mqtt, BName, Acc);
+get_matched_bridge_id(kafka, #{producer := #{mqtt := #{topic := Filter}}}, Topic, BName, Acc) ->
+    do_get_matched_bridge_id(Topic, Filter, kafka, BName, Acc).
+
+do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) ->
     case emqx_topic:match(Topic, Filter) of
         true -> [emqx_bridge_resource:bridge_id(BType, BName) | Acc];
         false -> Acc
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index e48833f78..5fced2467 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -42,8 +42,6 @@
 
 -export([lookup_from_local_node/2]).
 
--define(CONN_TYPES, [mqtt]).
-
 -define(TRY_PARSE_ID(ID, EXPR),
     try emqx_bridge_resource:parse_bridge_id(Id) of
         {BridgeType, BridgeName} ->
@@ -146,7 +144,7 @@ param_path_id() ->
             #{
                 in => path,
                 required => true,
-                example => <<"webhook:my_webhook">>,
+                example => <<"webhook:webhook_example">>,
                 desc => ?DESC("desc_param_path_id")
             }
         )}.
@@ -155,66 +153,45 @@ bridge_info_array_example(Method) ->
     [Config || #{value := Config} <- maps:values(bridge_info_examples(Method))].
 
 bridge_info_examples(Method) ->
-    maps:merge(conn_bridge_examples(Method), #{
-        <<"my_webhook">> => #{
-            summary => <<"WebHook">>,
-            value => info_example(webhook, awesome, Method)
-        }
-    }).
-
-conn_bridge_examples(Method) ->
-    Fun =
-        fun(Type, Acc) ->
-            SType = atom_to_list(Type),
-            KeyIngress = bin(SType ++ "_ingress"),
-            KeyEgress = bin(SType ++ "_egress"),
-            maps:merge(Acc, #{
-                KeyIngress => #{
-                    summary => bin(string:uppercase(SType) ++ " Ingress Bridge"),
-                    value => info_example(Type, ingress, Method)
-                },
-                KeyEgress => #{
-                    summary => bin(string:uppercase(SType) ++ " Egress Bridge"),
-                    value => info_example(Type, egress, Method)
-                }
-            })
-        end,
-    Broker = lists:foldl(Fun, #{}, ?CONN_TYPES),
-    EE = ee_conn_bridge_examples(Method),
-    maps:merge(Broker, EE).
-
--if(?EMQX_RELEASE_EDITION == ee).
-ee_conn_bridge_examples(Method) ->
-    emqx_ee_bridge:conn_bridge_examples(Method).
--else.
-ee_conn_bridge_examples(_Method) ->
-    #{}.
--endif.
-
-info_example(Type, Direction, Method) ->
     maps:merge(
-        info_example_basic(Type, Direction),
-        method_example(Type, Direction, Method)
+        #{
+            <<"webhook_example">> => #{
+                summary => <<"WebHook">>,
+                value => info_example(webhook, Method)
+            },
+            <<"mqtt_example">> => #{
+                summary => <<"MQTT Bridge">>,
+                value => info_example(mqtt, Method)
+            }
+        },
+        ee_bridge_examples(Method)
     ).
 
-method_example(Type, Direction, Method) when Method == get; Method == post ->
+ee_bridge_examples(Method) ->
+    case erlang:function_exported(emqx_ee_bridge, examples, 1) of
+        true -> emqx_ee_bridge:examples(Method);
+        false -> #{}
+    end.
+
+info_example(Type, Method) ->
+    maps:merge(
+        info_example_basic(Type),
+        method_example(Type, Method)
+    ).
+
+method_example(Type, Method) when Method == get; Method == post ->
     SType = atom_to_list(Type),
-    SDir = atom_to_list(Direction),
-    SName =
-        case Type of
-            webhook -> "my_" ++ SType;
-            _ -> "my_" ++ SDir ++ "_" ++ SType ++ "_bridge"
-        end,
-    TypeNameExamp = #{
+    SName = SType ++ "_example",
+    TypeNameExam = #{
         type => bin(SType),
         name => bin(SName)
     },
-    maybe_with_metrics_example(TypeNameExamp, Method);
-method_example(_Type, _Direction, put) ->
+    maybe_with_metrics_example(TypeNameExam, Method);
+method_example(_Type, put) ->
     #{}.
 
-maybe_with_metrics_example(TypeNameExamp, get) ->
-    TypeNameExamp#{
+maybe_with_metrics_example(TypeNameExam, get) ->
+    TypeNameExam#{
         metrics => ?METRICS(0, 0, 0, 0, 0, 0),
         node_metrics => [
             #{
@@ -223,10 +200,10 @@ maybe_with_metrics_example(TypeNameExamp, get) ->
             }
         ]
     };
-maybe_with_metrics_example(TypeNameExamp, _) ->
-    TypeNameExamp.
+maybe_with_metrics_example(TypeNameExam, _) ->
+    TypeNameExam.
 
-info_example_basic(webhook, _) ->
+info_example_basic(webhook) ->
     #{
         enable => true,
         url => <<"http://localhost:9901/messages/${topic}">>,
@@ -241,28 +218,52 @@ info_example_basic(webhook, _) ->
         method => post,
         body => <<"${payload}">>
     };
-info_example_basic(mqtt, ingress) ->
+info_example_basic(mqtt) ->
+    (mqtt_main_example())#{
+        egress => mqtt_egress_example(),
+        ingress => mqtt_ingress_example()
+    }.
+
+mqtt_main_example() ->
     #{
         enable => true,
-        connector => <<"mqtt:my_mqtt_connector">>,
-        direction => ingress,
-        remote_topic => <<"aws/#">>,
-        remote_qos => 1,
-        local_topic => <<"from_aws/${topic}">>,
-        local_qos => <<"${qos}">>,
-        payload => <<"${payload}">>,
-        retain => <<"${retain}">>
-    };
-info_example_basic(mqtt, egress) ->
+        mode => cluster_shareload,
+        server => <<"127.0.0.1:1883">>,
+        proto_ver => <<"v4">>,
+        username => <<"foo">>,
+        password => <<"bar">>,
+        clean_start => true,
+        keepalive => <<"300s">>,
+        retry_interval => <<"15s">>,
+        max_inflight => 100,
+        ssl => #{
+            enable => false
+        }
+    }.
+mqtt_egress_example() ->
     #{
-        enable => true,
-        connector => <<"mqtt:my_mqtt_connector">>,
-        direction => egress,
-        local_topic => <<"emqx/#">>,
-        remote_topic => <<"from_emqx/${topic}">>,
-        remote_qos => <<"${qos}">>,
-        payload => <<"${payload}">>,
-        retain => false
+        local => #{
+            topic => <<"emqx/#">>
+        },
+        remote => #{
+            topic => <<"from_emqx/${topic}">>,
+            qos => <<"${qos}">>,
+            payload => <<"${payload}">>,
+            retain => false
+        }
+    }.
+mqtt_ingress_example() ->
+    #{
+        remote => #{
+            topic => <<"aws/#">>,
+            qos => 1
+        },
+        local => #{
+            topic => <<"from_aws/${topic}">>,
+            qos => <<"${qos}">>,
+            payload => <<"${payload}">>,
+            retain => <<"${retain}">>
+        }
     }.
 
 schema("/bridges") ->
diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl
index cac6ab1e6..46f5d6729 100644
--- a/apps/emqx_bridge/src/emqx_bridge_app.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_app.erl
@@ -45,7 +45,6 @@ stop(_State) ->
 -if(?EMQX_RELEASE_EDITION == ee).
 start_ee_apps() ->
     {ok, _} = application:ensure_all_started(emqx_ee_bridge),
-    {ok, _} = application:ensure_all_started(emqx_ee_connector),
     ok.
 -else.
 start_ee_apps() ->
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index f7aeec30d..f0773a8ea 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -43,6 +43,9 @@
     reset_metrics/1
 ]).
 
+%% bi-directional bridge with producer/consumer or ingress/egress configs
+-define(IS_BI_DIR_BRIDGE(TYPE), TYPE == <<"mqtt">>; TYPE == <<"kafka">>).
+
 -if(?EMQX_RELEASE_EDITION == ee).
 bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt;
 bridge_to_resource_type(mqtt) -> emqx_connector_mqtt;
@@ -102,7 +105,7 @@ create(Type, Name, Conf, Opts) ->
         resource_id(Type, Name),
         <<"emqx_bridge">>,
         bridge_to_resource_type(Type),
-        parse_confs(Type, Name, Conf),
+        parse_confs(bin(Type), Name, Conf),
         Opts
     ),
     maybe_disable_bridge(Type, Name, Conf).
@@ -168,27 +171,21 @@ recreate(Type, Name, Conf, Opts) ->
     emqx_resource:recreate_local(
         resource_id(Type, Name),
         bridge_to_resource_type(Type),
-        parse_confs(Type, Name, Conf),
+        parse_confs(bin(Type), Name, Conf),
         Opts
     ).
 
 create_dry_run(Type, Conf) ->
-    Conf0 = fill_dry_run_conf(Conf),
-    case emqx_resource:check_config(bridge_to_resource_type(Type), Conf0) of
-        {ok, Conf1} ->
-            TmpPath = iolist_to_binary(["bridges-create-dry-run:", emqx_misc:gen_id(8)]),
-            case emqx_connector_ssl:convert_certs(TmpPath, Conf1) of
-                {error, Reason} ->
-                    {error, Reason};
-                {ok, ConfNew} ->
-                    Res = emqx_resource:create_dry_run_local(
-                        bridge_to_resource_type(Type), ConfNew
-                    ),
-                    _ = maybe_clear_certs(TmpPath, ConfNew),
-                    Res
-            end;
-        {error, _} = Error ->
-            Error
+    TmpPath = iolist_to_binary(["bridges-create-dry-run:", emqx_misc:gen_id(8)]),
+    case emqx_connector_ssl:convert_certs(TmpPath, Conf) of
+        {error, Reason} ->
+            {error, Reason};
+        {ok, ConfNew} ->
+            Res = emqx_resource:create_dry_run_local(
+                bridge_to_resource_type(Type), ConfNew
+            ),
+            _ = maybe_clear_certs(TmpPath, ConfNew),
+            Res
     end.
 
 remove(BridgeId) ->
@@ -213,19 +210,6 @@ maybe_disable_bridge(Type, Name, Conf) ->
         true -> ok
     end.
 
-fill_dry_run_conf(Conf) ->
-    Conf#{
-        <<"egress">> =>
-            #{
-                <<"remote_topic">> => <<"t">>,
-                <<"remote_qos">> => 0,
-                <<"retain">> => true,
-                <<"payload">> => <<"val">>
-            },
-        <<"ingress">> =>
-            #{<<"remote_topic">> => <<"t">>}
-    }.
-
 maybe_clear_certs(TmpPath, #{ssl := SslConf} = Conf) ->
     %% don't remove the cert files if they are in use
     case is_tmp_path_conf(TmpPath, SslConf) of
@@ -245,8 +229,9 @@ is_tmp_path_conf(_TmpPath, _Conf) ->
 is_tmp_path(TmpPath, File) ->
     string:str(str(File), str(TmpPath)) > 0.
 
+%% convert bridge configs to what the connector modules want
 parse_confs(
-    Type,
+    <<"webhook">>,
     _Name,
     #{
         url := Url,
@@ -256,7 +241,7 @@ parse_confs(
         request_timeout := ReqTimeout,
         max_retries := Retry
     } = Conf
-) when Type == webhook orelse Type == <<"webhook">> ->
+) ->
     {BaseUrl, Path} = parse_url(Url),
     {ok, BaseUrl2} = emqx_http_lib:uri_parse(BaseUrl),
     Conf#{
@@ -271,42 +256,14 @@ parse_confs(
                 max_retries => Retry
             }
     };
-parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) when
-    is_binary(ConnId)
-->
-    case emqx_connector:parse_connector_id(ConnId) of
-        {Type, ConnName} ->
-            ConnectorConfs = emqx:get_config([connectors, Type, ConnName]),
-            make_resource_confs(
-                Direction,
-                ConnectorConfs,
-                maps:without([connector, direction], Conf),
-                Type,
-                Name
-            );
-        {_ConnType, _ConnName} ->
-            error({cannot_use_connector_with_different_type, ConnId})
-    end;
-parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when
-    is_map(ConnectorConfs)
-->
-    make_resource_confs(
-        Direction,
-        ConnectorConfs,
-        maps:without([connector, direction], Conf),
-        Type,
-        Name
-    ).
-
-make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
+parse_confs(Type, Name, Conf) when ?IS_BI_DIR_BRIDGE(Type) ->
+    %% For some drivers that can be used as data-sources, we need to provide a
+    %% hookpoint. The underlying driver will run `emqx_hooks:run/3` when it
+    %% receives a message from the external database.
     BName = bridge_id(Type, Name),
-    ConnectorConfs#{
-        ingress => BridgeConf#{hookpoint => <<"$bridges/", BName/binary>>}
-    };
-make_resource_confs(egress, ConnectorConfs, BridgeConf, _Type, _Name) ->
-    ConnectorConfs#{
-        egress => BridgeConf
-    }.
+    Conf#{hookpoint => <<"$bridges/", BName/binary>>};
+parse_confs(_Type, _Name, Conf) ->
+    Conf.
 
 parse_url(Url) ->
     case string:split(Url, "//", leading) of
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
index 9fc06ec0e..1bf5565e9 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
@@ -3,43 +3,28 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
--import(hoconsc, [mk/2]).
+-import(hoconsc, [mk/2, ref/2]).
 
--export([roots/0, fields/1, desc/1]).
+-export([roots/0, fields/1, desc/1, namespace/0]).
 
 %%======================================================================================
 %% Hocon Schema Definitions
+namespace() -> "bridge_mqtt".
+
 roots() -> [].
-
-fields("ingress") ->
-    [emqx_bridge_schema:direction_field(ingress, emqx_connector_mqtt_schema:ingress_desc())] ++
-        emqx_bridge_schema:common_bridge_fields(mqtt_connector_ref()) ++
-        proplists:delete(hookpoint, emqx_connector_mqtt_schema:fields("ingress"));
-fields("egress") ->
-    [emqx_bridge_schema:direction_field(egress, emqx_connector_mqtt_schema:egress_desc())] ++
-        emqx_bridge_schema:common_bridge_fields(mqtt_connector_ref()) ++
-        emqx_connector_mqtt_schema:fields("egress");
-fields("post_ingress") ->
+fields("config") ->
+    emqx_bridge_schema:common_bridge_fields() ++
+        emqx_connector_mqtt_schema:fields("config");
+fields("post") ->
     [
         type_field(),
         name_field()
-    ] ++ proplists:delete(enable, fields("ingress"));
-fields("post_egress") ->
-    [
-        type_field(),
-        name_field()
-    ] ++ proplists:delete(enable, fields("egress"));
-fields("put_ingress") ->
-    proplists:delete(enable, fields("ingress"));
-fields("put_egress") ->
-    proplists:delete(enable, fields("egress"));
-fields("get_ingress") ->
-    emqx_bridge_schema:metrics_status_fields() ++ fields("post_ingress");
-fields("get_egress") ->
-    emqx_bridge_schema:metrics_status_fields() ++ fields("post_egress").
+    ] ++ emqx_connector_mqtt_schema:fields("config");
+fields("put") ->
+    emqx_connector_mqtt_schema:fields("config");
+fields("get") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields("config").
 
-desc(Rec) when Rec =:= "ingress"; Rec =:= "egress" ->
-    ?DESC("desc_rec");
 desc(_) ->
     undefined.
 
@@ -63,6 +48,3 @@ name_field() ->
                 desc => ?DESC("desc_name")
             }
         )}.
-
-mqtt_connector_ref() ->
-    ?R_REF(emqx_connector_mqtt_schema, "connector").
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 4343dc223..aedfcaa03 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -14,16 +14,13 @@
 ]).
 
 -export([
-    common_bridge_fields/1,
-    metrics_status_fields/0,
-    direction_field/2
+    common_bridge_fields/0,
+    metrics_status_fields/0
 ]).
 
 %%======================================================================================
 %% Hocon Schema Definitions
 
--define(CONN_TYPES, [mqtt]).
-
 %%======================================================================================
 %% For HTTP APIs
 get_response() ->
@@ -36,34 +33,26 @@ post_request() ->
     api_schema("post").
 
 api_schema(Method) ->
-    Broker =
-        lists:flatmap(
-            fun(Type) ->
-                [
-                    ref(schema_mod(Type), Method ++ "_ingress"),
-                    ref(schema_mod(Type), Method ++ "_egress")
-                ]
-            end,
-            ?CONN_TYPES
-        ) ++ [ref(Module, Method) || Module <- [emqx_bridge_webhook_schema]],
+    Broker = [
+        ref(Mod, Method)
+     || Mod <- [emqx_bridge_webhook_schema, emqx_bridge_mqtt_schema]
+    ],
     EE = ee_api_schemas(Method),
     hoconsc:union(Broker ++ EE).
 
--if(?EMQX_RELEASE_EDITION == ee).
 ee_api_schemas(Method) ->
-    emqx_ee_bridge:api_schemas(Method).
+    case erlang:function_exported(emqx_ee_bridge, api_schemas, 1) of
+        true -> emqx_ee_bridge:api_schemas(Method);
+        false -> []
+    end.
 
 ee_fields_bridges() ->
-    emqx_ee_bridge:fields(bridges).
--else.
-ee_api_schemas(_) ->
-    [].
+    case erlang:function_exported(emqx_ee_bridge, fields, 1) of
+        true -> emqx_ee_bridge:fields(bridges);
+        false -> []
+    end.
 
-ee_fields_bridges() ->
-    [].
--endif.
-
-common_bridge_fields(ConnectorRef) ->
+common_bridge_fields() ->
     [
         {enable,
             mk(
@@ -72,15 +61,6 @@ common_bridge_fields(ConnectorRef) ->
                     desc => ?DESC("desc_enable"),
                     default => true
                 }
-            )},
-        {connector,
-            mk(
-                hoconsc:union([binary(), ConnectorRef]),
-                #{
-                    required => true,
-                    example => <<"mqtt:my_mqtt_connector">>,
-                    desc => ?DESC("desc_connector")
-                }
             )}
     ].
 
@@ -100,18 +80,6 @@ metrics_status_fields() ->
             )}
     ].
 
-direction_field(Dir, Desc) ->
-    {direction,
-        mk(
-            Dir,
-            #{
-                required => true,
-                default => egress,
-                desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.</br>" ++
-                    Desc
-            }
-        )}.
-
 %%======================================================================================
 %% For config files
 
@@ -125,22 +93,13 @@ fields(bridges) ->
             mk(
                 hoconsc:map(name, ref(emqx_bridge_webhook_schema, "config")),
                 #{desc => ?DESC("bridges_webhook")}
+            )},
+        {mqtt,
+            mk(
+                hoconsc:map(name, ref(emqx_bridge_mqtt_schema, "config")),
+                #{desc => ?DESC("bridges_mqtt")}
             )}
-    ] ++
-        [
-            {T,
-                mk(
-                    hoconsc:map(
-                        name,
-                        hoconsc:union([
-                            ref(schema_mod(T), "ingress"),
-                            ref(schema_mod(T), "egress")
-                        ])
-                    ),
-                    #{desc => ?DESC("bridges_name")}
-                )}
-         || T <- ?CONN_TYPES
-        ] ++ ee_fields_bridges();
+    ] ++ ee_fields_bridges();
 fields("metrics") ->
     [
         {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})},
@@ -181,6 +140,3 @@ status() ->
 
 node_name() ->
     {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}.
-
-schema_mod(Type) ->
-    list_to_atom(lists:concat(["emqx_bridge_", Type, "_schema"])).
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
index 7c19e9e55..0f692f195 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
@@ -50,14 +50,6 @@ basic_config() ->
                     desc => ?DESC("config_enable"),
                     default => true
                 }
-            )},
-        {direction,
-            mk(
-                egress,
-                #{
-                    desc => ?DESC("config_direction"),
-                    default => egress
-                }
             )}
     ] ++ webhook_creation_opts() ++
         proplists:delete(
diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl
index ce54695a5..25aa82d76 100644
--- a/apps/emqx_conf/src/emqx_conf.erl
+++ b/apps/emqx_conf/src/emqx_conf.erl
@@ -165,7 +165,6 @@ gen_schema_json(Dir, I18nFile, SchemaModule) ->
 gen_api_schema_json(Dir, I18nFile, Lang) ->
     emqx_dashboard:init_i18n(I18nFile, Lang),
     gen_api_schema_json_hotconf(Dir, Lang),
-    gen_api_schema_json_connector(Dir, Lang),
     gen_api_schema_json_bridge(Dir, Lang),
     emqx_dashboard:clear_i18n().
 
@@ -174,11 +173,6 @@ gen_api_schema_json_hotconf(Dir, Lang) ->
     File = schema_filename(Dir, "hot-config-schema-", Lang),
     ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo).
 
-gen_api_schema_json_connector(Dir, Lang) ->
-    SchemaInfo = #{title => <<"EMQX Connector API Schema">>, version => <<"0.1.0">>},
-    File = schema_filename(Dir, "connector-api-", Lang),
-    ok = do_gen_api_schema_json(File, emqx_connector_api, SchemaInfo).
-
 gen_api_schema_json_bridge(Dir, Lang) ->
     SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>},
     File = schema_filename(Dir, "bridge-api-", Lang),
diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index 1b24a0a38..726416568 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -60,7 +60,6 @@
     emqx_exhook_schema,
     emqx_psk_schema,
     emqx_limiter_schema,
-    emqx_connector_schema,
     emqx_slow_subs_schema
 ]).
 
diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt.conf
index 1005d68dc..5ade54670 100644
--- a/apps/emqx_connector/i18n/emqx_connector_mqtt.conf
+++ b/apps/emqx_connector/i18n/emqx_connector_mqtt.conf
@@ -1,5 +1,4 @@
 emqx_connector_mqtt {
-
     num_of_bridges {
         desc {
           en: "The current number of bridges that are using this connector."
diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
index c9f227383..6dfd0ef27 100644
--- a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
+++ b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
@@ -1,4 +1,85 @@
 emqx_connector_mqtt_schema {
+    ingress_desc {
+        desc {
+                en: """The ingress config defines how this bridge receive messages from the remote MQTT broker, and then
+        send them to the local broker.</br>
+        Template with variables is allowed in 'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'.</br>
+        NOTE: if this bridge is used as the input of a rule, and also 'local.topic' is
+        configured, then messages got from the remote broker will be sent to both the 'local.topic' and
+        the rule."""
+                zh: """入口配置定义了该桥接如何从远程 MQTT Broker 接收消息，然后将消息发送到本地 Broker。</br>
+        以下字段中允许使用带有变量的模板：'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'。</br>
+        注意：如果此桥接被用作规则的输入，并且配置了 'local.topic'，则从远程代理获取的消息将同时被发送到 'local.topic' 和规则。
+                """
+        }
+        label: {
+                en: "Ingress Configs"
+                zh: "入方向配置"
+        }
+    }
+
+    egress_desc {
+        desc {
+                en: """The egress config defines how this bridge forwards messages from the local broker to the remote broker.</br>
+Template with variables is allowed in 'remote.topic', 'local.qos', 'local.retain', 'local.payload'.</br>
+NOTE: if this bridge is used as the action of a rule, and also 'local.topic'
+is configured, then both the data got from the rule and the MQTT messages that matches
+'local.topic' will be forwarded."""
+                zh: """出口配置定义了该桥接如何将消息从本地 Broker 转发到远程 Broker。
+以下字段中允许使用带有变量的模板：'remote.topic', 'local.qos', 'local.retain', 'local.payload'。</br>
+注意：如果此桥接被用作规则的动作，并且配置了 'local.topic'，则从规则输出的数据以及匹配到 'local.topic' 的 MQTT 消息都会被转发。
+                """
+        }
+        label: {
+                en: "Egress Configs"
+                zh: "出方向配置"
+        }
+    }
+
+    ingress_remote {
+        desc {
+                en: """The configs about subscribing to the remote broker."""
+                zh: """订阅远程 Broker 相关的配置。"""
+        }
+        label: {
+                en: "Remote Configs"
+                zh: "远程配置"
+        }
+    }
+
+    ingress_local {
+        desc {
+                en: """The configs about sending message to the local broker."""
+                zh: """发送消息到本地 Broker 相关的配置。"""
+        }
+        label: {
+                en: "Local Configs"
+                zh: "本地配置"
+        }
+    }
+
+    egress_remote {
+        desc {
+                en: """The configs about sending message to the remote broker."""
+                zh: """发送消息到远程 Broker 相关的配置。"""
+        }
+        label: {
+                en: "Remote Configs"
+                zh: "远程配置"
+        }
+    }
+
+    egress_local {
+        desc {
+                en: """The configs about receiving messages from ben."""
+                zh: """收取本地 Broker 消息相关的配置。"""
+        }
+        label: {
+                en: "Local Configs"
+                zh: "本地配置"
+        }
+    }
+
     mode {
         desc {
           en: """
@@ -9,7 +90,7 @@ In 'cluster_shareload' mode, the incoming load from the remote broker is shared
 using shared subscription.</br>
 Note that the 'clientid' is suffixed by the node name, this is to avoid
 clientid conflicts between different nodes. And we can only use shared subscription
-topic filters for 'remote_topic' of ingress connections.
+topic filters for 'remote.topic' of ingress connections.
 """
           zh: """
 MQTT 桥的模式。 </br>
@@ -17,7 +98,7 @@ MQTT 桥的模式。 </br>
 - cluster_shareload：在 emqx 集群的每个节点上创建一个 MQTT 连接。</br>
 在“cluster_shareload”模式下，来自远程代理的传入负载通过共享订阅的方式接收。</br>
 请注意，“clientid”以节点名称为后缀，这是为了避免不同节点之间的clientid冲突。
-而且对于入口连接的“remote_topic”，我们只能使用共享订阅主题过滤器。
+而且对于入口连接的“remote.topic”，我们只能使用共享订阅主题过滤器。
 """
         }
         label: {
@@ -166,17 +247,6 @@ Template with variables is allowed.
             }
     }
 
-    ingress_hookpoint {
-        desc {
-          en: "The hook point will be triggered when there's any message received from the remote broker."
-          zh: "当从远程borker收到任何消息时，将触发钩子。"
-        }
-        label: {
-              en: "Hookpoint"
-              zh: "挂载点"
-            }
-    }
-
     egress_local_topic {
         desc {
           en: "The local topic to be forwarded to the remote broker"
@@ -222,59 +292,6 @@ Template with variables is allowed.
             }
     }
 
-    dir {
-        desc {
-          en: """
-The dir where the replayq file saved.</br>
-Set to 'false' disables the replayq feature.
-"""
-          zh: """
-replayq 文件保存的目录。</br>
-设置为 'false' 会禁用 replayq 功能。
-"""
-        }
-        label: {
-              en: "Replyq file Save Dir"
-              zh: "Replyq 文件保存目录"
-            }
-    }
-
-    seg_bytes {
-        desc {
-          en: """
-The size in bytes of a single segment.</br>
-A segment is mapping to a file in the replayq dir. If the current segment is full, a new segment
-(file) will be opened to write.
-"""
-          zh: """
-单个段的大小（以字节为单位）。</br>
-一个段映射到 replayq 目录中的一个文件。 如果当前段已满，则新段（文件）将被打开写入。
-"""
-        }
-        label: {
-              en: "Segment Size"
-              zh: "Segment 大小"
-            }
-    }
-
-    offload {
-        desc {
-          en: """
-In offload mode, the disk queue is only used to offload queue tail segments.</br>
-The messages are cached in the memory first, then it writes to the replayq files after the size of
-the memory cache reaches 'seg_bytes'.
-"""
-          zh: """
-在Offload模式下，磁盘队列仅用于卸载队列尾段。</br>
-消息首先缓存在内存中，然后写入replayq文件。内存缓大小为“seg_bytes” 指定的值。
-"""
-        }
-        label: {
-              en: "Offload Mode"
-              zh: "Offload 模式"
-            }
-    }
-
     retain {
         desc {
           en: """
@@ -309,66 +326,15 @@ Template with variables is allowed.
             }
     }
 
-    desc_connector {
+    server_configs {
                    desc {
-                         en: """Generic configuration for the connector."""
-                         zh: """连接器的通用配置。"""
+                         en: """Configs related to the server."""
+                         zh: """服务器相关的配置。"""
                         }
                    label: {
-                           en: "Connector Generic Configuration"
-                           zh: "连接器通用配置。"
+                           en: "Server Configs"
+                           zh: "服务配置。"
                           }
                   }
 
-    desc_ingress {
-                   desc {
-                         en: """
-The ingress config defines how this bridge receive messages from the remote MQTT broker, and then send them to the local broker.</br>
-Template with variables is allowed in 'local_topic', 'remote_qos', 'qos', 'retain', 'payload'.</br>
-NOTE: if this bridge is used as the input of a rule (emqx rule engine), and also local_topic is configured, then messages got from the remote broker will be sent to both the 'local_topic' and the rule.
-"""
-                         zh: """
-Ingress 模式定义了这个 bridge 如何从远程 MQTT broker 接收消息，然后将它们发送到本地 broker 。</br>
-允许带有的模板变量: 'local_topic'、'remote_qos'、'qos'、'retain'、'payload' 。</br>
-注意：如果这个 bridge 被用作规则的输入（emqx 规则引擎），并且还配置了 local_topic，那么从远程 broker 获取的消息将同时被发送到 'local_topic' 和规则引擎。
-"""
-                        }
-                   label: {
-                           en: "Ingress Config"
-                           zh: "Ingress 模式配置"
-                          }
-                  }
-
-    desc_egress {
-                   desc {
-                         en: """
-The egress config defines how this bridge forwards messages from the local broker to the remote broker.</br>
-Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.</br>
-NOTE: if this bridge is used as the action of a rule (emqx rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that matches local_topic will be forwarded.
-"""
-                         zh: """
-Egress 模式定义了 bridge 如何将消息从本地 broker 转发到远程 broker。</br>
-允许带有的模板变量: 'remote_topic'、'qos'、'retain'、'payload' 。</br>
-注意：如果这个 bridge 作为规则（emqx 规则引擎）的输出，并且还配置了 local_topic，那么从规则引擎中获取的数据和匹配 local_topic 的 MQTT 消息都会被转发到远程 broker 。
-"""
-                        }
-                   label: {
-                           en: "Egress Config"
-                           zh: "Egress 模式配置"
-                          }
-                  }
-
-    desc_replayq {
-                   desc {
-                         en: """Queue messages in disk files."""
-                         zh: """本地磁盘消息队列"""
-                        }
-                   label: {
-                           en: "Replayq"
-                           zh: "本地磁盘消息队列"
-                          }
-                  }
-
-
-
 }
diff --git a/apps/emqx_connector/i18n/emqx_connector_schema.conf b/apps/emqx_connector/i18n/emqx_connector_schema.conf
deleted file mode 100644
index 1f6fd5381..000000000
--- a/apps/emqx_connector/i18n/emqx_connector_schema.conf
+++ /dev/null
@@ -1,31 +0,0 @@
-emqx_connector_schema {
-
-    mqtt {
-        desc {
-          en: "MQTT bridges."
-          zh: "MQTT bridges。"
-        }
-        label: {
-              en: "MQTT bridges"
-              zh: "MQTT bridges"
-            }
-    }
-
-    desc_connector {
-        desc {
-          en: """
-Configuration for EMQX connectors.</br>
-A connector maintains the data related to the external resources, such as MySQL database.
-"""
-          zh: """
-EMQX 连接器的配置。</br>
-连接器维护与外部资源相关的数据，比如 MySQL 数据库。
-"""
-        }
-        label: {
-              en: "Connector"
-              zh: "连接器"
-            }
-    }
-
-}
diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl
deleted file mode 100644
index d85959698..000000000
--- a/apps/emqx_connector/src/emqx_connector.erl
+++ /dev/null
@@ -1,165 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
--module(emqx_connector).
-
--export([
-    config_key_path/0,
-    pre_config_update/3,
-    post_config_update/5
-]).
-
--export([
-    parse_connector_id/1,
-    connector_id/2
-]).
-
--export([
-    list_raw/0,
-    lookup_raw/1,
-    lookup_raw/2,
-    create_dry_run/2,
-    update/2,
-    update/3,
-    delete/1,
-    delete/2
-]).
-
-config_key_path() ->
-    [connectors].
-
-pre_config_update(Path, Conf, _OldConfig) when is_map(Conf) ->
-    emqx_connector_ssl:convert_certs(filename:join(Path), Conf).
-
--dialyzer([{nowarn_function, [post_config_update/5]}, error_handling]).
-post_config_update([connectors, Type, Name] = Path, '$remove', _, OldConf, _AppEnvs) ->
-    ConnId = connector_id(Type, Name),
-    try
-        foreach_linked_bridges(ConnId, fun(#{type := BType, name := BName}) ->
-            throw({dependency_bridges_exist, emqx_bridge_resource:bridge_id(BType, BName)})
-        end),
-        _ = emqx_connector_ssl:clear_certs(filename:join(Path), OldConf)
-    catch
-        throw:Error -> {error, Error}
-    end;
-post_config_update([connectors, Type, Name], _Req, NewConf, OldConf, _AppEnvs) ->
-    ConnId = connector_id(Type, Name),
-    foreach_linked_bridges(
-        ConnId,
-        fun(#{type := BType, name := BName}) ->
-            BridgeConf = emqx:get_config([bridges, BType, BName]),
-            case
-                emqx_bridge_resource:update(
-                    BType,
-                    BName,
-                    {BridgeConf#{connector => OldConf}, BridgeConf#{connector => NewConf}}
-                )
-            of
-                ok -> ok;
-                {error, Reason} -> error({update_bridge_error, Reason})
-            end
-        end
-    ).
-
-connector_id(Type0, Name0) ->
-    Type = bin(Type0),
-    Name = bin(Name0),
-    <<Type/binary, ":", Name/binary>>.
-
-parse_connector_id(ConnectorId) ->
-    case string:split(bin(ConnectorId), ":", all) of
-        [Type, Name] -> {binary_to_atom(Type, utf8), binary_to_atom(Name, utf8)};
-        _ -> error({invalid_connector_id, ConnectorId})
-    end.
-
-list_raw() ->
-    case get_raw_connector_conf() of
-        not_found ->
-            [];
-        Config ->
-            lists:foldl(
-                fun({Type, NameAndConf}, Connectors) ->
-                    lists:foldl(
-                        fun({Name, RawConf}, Acc) ->
-                            [RawConf#{<<"type">> => Type, <<"name">> => Name} | Acc]
-                        end,
-                        Connectors,
-                        maps:to_list(NameAndConf)
-                    )
-                end,
-                [],
-                maps:to_list(Config)
-            )
-    end.
-
-lookup_raw(Id) when is_binary(Id) ->
-    {Type, Name} = parse_connector_id(Id),
-    lookup_raw(Type, Name).
-
-lookup_raw(Type, Name) ->
-    Path = [bin(P) || P <- [Type, Name]],
-    case get_raw_connector_conf() of
-        not_found ->
-            {error, not_found};
-        Conf ->
-            case emqx_map_lib:deep_get(Path, Conf, not_found) of
-                not_found -> {error, not_found};
-                Conf1 -> {ok, Conf1#{<<"type">> => Type, <<"name">> => Name}}
-            end
-    end.
-
--spec create_dry_run(module(), binary() | #{binary() => term()} | [#{binary() => term()}]) ->
-    ok | {error, Reason :: term()}.
-create_dry_run(Type, Conf) ->
-    emqx_bridge_resource:create_dry_run(Type, Conf).
-
-update(Id, Conf) when is_binary(Id) ->
-    {Type, Name} = parse_connector_id(Id),
-    update(Type, Name, Conf).
-
-update(Type, Name, Conf) ->
-    emqx_conf:update(config_key_path() ++ [Type, Name], Conf, #{override_to => cluster}).
-
-delete(Id) when is_binary(Id) ->
-    {Type, Name} = parse_connector_id(Id),
-    delete(Type, Name).
-
-delete(Type, Name) ->
-    emqx_conf:remove(config_key_path() ++ [Type, Name], #{override_to => cluster}).
-
-get_raw_connector_conf() ->
-    case emqx:get_raw_config(config_key_path(), not_found) of
-        not_found ->
-            not_found;
-        RawConf ->
-            #{<<"connectors">> := Conf} =
-                emqx_config:fill_defaults(#{<<"connectors">> => RawConf}),
-            Conf
-    end.
-
-bin(Bin) when is_binary(Bin) -> Bin;
-bin(Str) when is_list(Str) -> list_to_binary(Str);
-bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
-
-foreach_linked_bridges(ConnId, Do) ->
-    lists:foreach(
-        fun
-            (#{raw_config := #{<<"connector">> := ConnId0}} = Bridge) when ConnId0 == ConnId ->
-                Do(Bridge);
-            (_) ->
-                ok
-        end,
-        emqx_bridge:list()
-    ).
diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl
deleted file mode 100644
index 8dcd3a4aa..000000000
--- a/apps/emqx_connector/src/emqx_connector_api.erl
+++ /dev/null
@@ -1,339 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_connector_api).
-
--behaviour(minirest_api).
-
--include("emqx_connector.hrl").
-
--include_lib("typerefl/include/types.hrl").
--include_lib("hocon/include/hoconsc.hrl").
-
--import(hoconsc, [mk/2, ref/2, array/1, enum/1]).
-
-%% Swagger specs from hocon schema
--export([api_spec/0, paths/0, schema/1, namespace/0]).
-
-%% API callbacks
--export(['/connectors_test'/2, '/connectors'/2, '/connectors/:id'/2]).
-
--define(CONN_TYPES, [mqtt]).
-
--define(TRY_PARSE_ID(ID, EXPR),
-    try emqx_connector:parse_connector_id(Id) of
-        {ConnType, ConnName} ->
-            _ = ConnName,
-            EXPR
-    catch
-        error:{invalid_connector_id, Id0} ->
-            {400, #{
-                code => 'INVALID_ID',
-                message =>
-                    <<"invalid_connector_id: ", Id0/binary,
-                        ". Connector Ids must be of format {type}:{name}">>
-            }}
-    end
-).
-
-namespace() -> "connector".
-
-api_spec() ->
-    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}).
-
-paths() -> ["/connectors_test", "/connectors", "/connectors/:id"].
-
-error_schema(Codes, Message) when is_list(Message) ->
-    error_schema(Codes, list_to_binary(Message));
-error_schema(Codes, Message) when is_binary(Message) ->
-    emqx_dashboard_swagger:error_codes(Codes, Message).
-
-put_request_body_schema() ->
-    emqx_dashboard_swagger:schema_with_examples(
-        emqx_connector_schema:put_request(), connector_info_examples(put)
-    ).
-
-post_request_body_schema() ->
-    emqx_dashboard_swagger:schema_with_examples(
-        emqx_connector_schema:post_request(), connector_info_examples(post)
-    ).
-
-get_response_body_schema() ->
-    emqx_dashboard_swagger:schema_with_examples(
-        emqx_connector_schema:get_response(), connector_info_examples(get)
-    ).
-
-connector_info_array_example(Method) ->
-    [Config || #{value := Config} <- maps:values(connector_info_examples(Method))].
-
-connector_info_examples(Method) ->
-    Fun =
-        fun(Type, Acc) ->
-            SType = atom_to_list(Type),
-            maps:merge(Acc, #{
-                Type => #{
-                    summary => bin(string:uppercase(SType) ++ " Connector"),
-                    value => info_example(Type, Method)
-                }
-            })
-        end,
-    Broker = lists:foldl(Fun, #{}, ?CONN_TYPES),
-    EE = ee_example(Method),
-    maps:merge(Broker, EE).
-
--if(?EMQX_RELEASE_EDITION == ee).
-ee_example(Method) ->
-    emqx_ee_connector:connector_examples(Method).
--else.
-ee_example(_Method) ->
-    #{}.
--endif.
-
-info_example(Type, Method) ->
-    maps:merge(
-        info_example_basic(Type),
-        method_example(Type, Method)
-    ).
-
-method_example(Type, Method) when Method == get; Method == post ->
-    SType = atom_to_list(Type),
-    SName = "my_" ++ SType ++ "_connector",
-    #{
-        type => bin(SType),
-        name => bin(SName)
-    };
-method_example(_Type, put) ->
-    #{}.
-
-info_example_basic(mqtt) ->
-    #{
-        mode => cluster_shareload,
-        server => <<"127.0.0.1:1883">>,
-        reconnect_interval => <<"15s">>,
-        proto_ver => <<"v4">>,
-        username => <<"foo">>,
-        password => <<"bar">>,
-        clientid => <<"foo">>,
-        clean_start => true,
-        keepalive => <<"300s">>,
-        retry_interval => <<"15s">>,
-        max_inflight => 100,
-        ssl => #{
-            enable => false
-        }
-    }.
-
-param_path_id() ->
-    [
-        {id,
-            mk(
-                binary(),
-                #{
-                    in => path,
-                    example => <<"mqtt:my_mqtt_connector">>,
-                    desc => ?DESC("id")
-                }
-            )}
-    ].
-
-schema("/connectors_test") ->
-    #{
-        'operationId' => '/connectors_test',
-        post => #{
-            tags => [<<"connectors">>],
-            desc => ?DESC("conn_test_post"),
-            summary => <<"Test creating connector">>,
-            'requestBody' => post_request_body_schema(),
-            responses => #{
-                204 => <<"Test connector OK">>,
-                400 => error_schema(['TEST_FAILED'], "connector test failed")
-            }
-        }
-    };
-schema("/connectors") ->
-    #{
-        'operationId' => '/connectors',
-        get => #{
-            tags => [<<"connectors">>],
-            desc => ?DESC("conn_get"),
-            summary => <<"List connectors">>,
-            responses => #{
-                200 => emqx_dashboard_swagger:schema_with_example(
-                    array(emqx_connector_schema:get_response()),
-                    connector_info_array_example(get)
-                )
-            }
-        },
-        post => #{
-            tags => [<<"connectors">>],
-            desc => ?DESC("conn_post"),
-            summary => <<"Create connector">>,
-            'requestBody' => post_request_body_schema(),
-            responses => #{
-                201 => get_response_body_schema(),
-                400 => error_schema(['ALREADY_EXISTS'], "connector already exists")
-            }
-        }
-    };
-schema("/connectors/:id") ->
-    #{
-        'operationId' => '/connectors/:id',
-        get => #{
-            tags => [<<"connectors">>],
-            desc => ?DESC("conn_id_get"),
-            summary => <<"Get connector">>,
-            parameters => param_path_id(),
-            responses => #{
-                200 => get_response_body_schema(),
-                404 => error_schema(['NOT_FOUND'], "Connector not found"),
-                400 => error_schema(['INVALID_ID'], "Bad connector ID")
-            }
-        },
-        put => #{
-            tags => [<<"connectors">>],
-            desc => ?DESC("conn_id_put"),
-            summary => <<"Update connector">>,
-            parameters => param_path_id(),
-            'requestBody' => put_request_body_schema(),
-            responses => #{
-                200 => get_response_body_schema(),
-                404 => error_schema(['NOT_FOUND'], "Connector not found"),
-                400 => error_schema(['INVALID_ID'], "Bad connector ID")
-            }
-        },
-        delete => #{
-            tags => [<<"connectors">>],
-            desc => ?DESC("conn_id_delete"),
-            summary => <<"Delete connector">>,
-            parameters => param_path_id(),
-            responses => #{
-                204 => <<"Delete connector successfully">>,
-                403 => error_schema(['DEPENDENCY_EXISTS'], "Cannot remove dependent connector"),
-                404 => error_schema(['NOT_FOUND'], "Delete failed, not found"),
-                400 => error_schema(['INVALID_ID'], "Bad connector ID")
-            }
-        }
-    }.
-
-'/connectors_test'(post, #{body := #{<<"type">> := ConnType} = Params}) ->
-    case emqx_connector:create_dry_run(ConnType, maps:remove(<<"type">>, Params)) of
-        ok ->
-            {204};
-        {error, Error} ->
-            {400, error_msg(['TEST_FAILED'], Error)}
-    end.
-
-'/connectors'(get, _Request) ->
-    {200, [format_resp(Conn) || Conn <- emqx_connector:list_raw()]};
-'/connectors'(post, #{body := #{<<"type">> := ConnType, <<"name">> := ConnName} = Params}) ->
-    case emqx_connector:lookup_raw(ConnType, ConnName) of
-        {ok, _} ->
-            {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)};
-        {error, not_found} ->
-            case
-                emqx_connector:update(
-                    ConnType,
-                    ConnName,
-                    filter_out_request_body(Params)
-                )
-            of
-                {ok, #{raw_config := RawConf}} ->
-                    {201,
-                        format_resp(RawConf#{
-                            <<"type">> => ConnType,
-                            <<"name">> => ConnName
-                        })};
-                {error, Error} ->
-                    {400, error_msg('BAD_REQUEST', Error)}
-            end
-    end;
-'/connectors'(post, _) ->
-    {400, error_msg('BAD_REQUEST', <<"missing some required fields: [name, type]">>)}.
-
-'/connectors/:id'(get, #{bindings := #{id := Id}}) ->
-    ?TRY_PARSE_ID(
-        Id,
-        case emqx_connector:lookup_raw(ConnType, ConnName) of
-            {ok, Conf} ->
-                {200, format_resp(Conf)};
-            {error, not_found} ->
-                {404, error_msg('NOT_FOUND', <<"connector not found">>)}
-        end
-    );
-'/connectors/:id'(put, #{bindings := #{id := Id}, body := Params0}) ->
-    Params = filter_out_request_body(Params0),
-    ?TRY_PARSE_ID(
-        Id,
-        case emqx_connector:lookup_raw(ConnType, ConnName) of
-            {ok, _} ->
-                case emqx_connector:update(ConnType, ConnName, Params) of
-                    {ok, #{raw_config := RawConf}} ->
-                        {200,
-                            format_resp(RawConf#{
-                                <<"type">> => ConnType,
-                                <<"name">> => ConnName
-                            })};
-                    {error, Error} ->
-                        {500, error_msg('INTERNAL_ERROR', Error)}
-                end;
-            {error, not_found} ->
-                {404, error_msg('NOT_FOUND', <<"connector not found">>)}
-        end
-    );
-'/connectors/:id'(delete, #{bindings := #{id := Id}}) ->
-    ?TRY_PARSE_ID(
-        Id,
-        case emqx_connector:lookup_raw(ConnType, ConnName) of
-            {ok, _} ->
-                case emqx_connector:delete(ConnType, ConnName) of
-                    {ok, _} ->
-                        {204};
-                    {error, {post_config_update, _, {dependency_bridges_exist, BridgeID}}} ->
-                        {403,
-                            error_msg(
-                                'DEPENDENCY_EXISTS',
-                                <<"Cannot remove the connector as it's in use by a bridge: ",
-                                    BridgeID/binary>>
-                            )};
-                    {error, Error} ->
-                        {500, error_msg('INTERNAL_ERROR', Error)}
-                end;
-            {error, not_found} ->
-                {404, error_msg('NOT_FOUND', <<"connector not found">>)}
-        end
-    ).
-
-error_msg(Code, Msg) ->
-    #{code => Code, message => emqx_misc:readable_error_msg(Msg)}.
-
-format_resp(#{<<"type">> := ConnType, <<"name">> := ConnName} = RawConf) ->
-    NumOfBridges = length(
-        emqx_bridge:list_bridges_by_connector(
-            emqx_connector:connector_id(ConnType, ConnName)
-        )
-    ),
-    RawConf#{
-        <<"type">> => ConnType,
-        <<"name">> => ConnName,
-        <<"num_of_bridges">> => NumOfBridges
-    }.
-
-filter_out_request_body(Conf) ->
-    ExtraConfs = [<<"clientid">>, <<"num_of_bridges">>, <<"type">>, <<"name">>],
-    maps:without(ExtraConfs, Conf).
-
-bin(S) when is_list(S) ->
-    list_to_binary(S).
diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl
index b6f5b8623..62167dc18 100644
--- a/apps/emqx_connector/src/emqx_connector_app.erl
+++ b/apps/emqx_connector/src/emqx_connector_app.erl
@@ -20,15 +20,10 @@
 
 -export([start/2, stop/1]).
 
--define(CONF_HDLR_PATH, (emqx_connector:config_key_path() ++ ['?', '?'])).
-
 start(_StartType, _StartArgs) ->
-    ok = emqx_config_handler:add_handler(?CONF_HDLR_PATH, emqx_connector),
-    emqx_connector_mqtt_worker:register_metrics(),
     emqx_connector_sup:start_link().
 
 stop(_State) ->
-    emqx_config_handler:remove_handler(?CONF_HDLR_PATH),
     ok.
 
 %% internal functions
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index e37f6a9a2..b57a94b36 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -67,7 +67,7 @@ fields("get") ->
             )}
     ] ++ fields("post");
 fields("put") ->
-    emqx_connector_mqtt_schema:fields("connector");
+    emqx_connector_mqtt_schema:fields("server_configs");
 fields("post") ->
     [
         {type,
@@ -236,11 +236,9 @@ basic_config(#{
     keepalive := KeepAlive,
     retry_interval := RetryIntv,
     max_inflight := MaxInflight,
-    replayq := ReplayQ,
     ssl := #{enable := EnableSsl} = Ssl
 }) ->
     #{
-        replayq => ReplayQ,
         %% connection opts
         server => Server,
         %% 30s
diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl
deleted file mode 100644
index f0c9479de..000000000
--- a/apps/emqx_connector/src/emqx_connector_schema.erl
+++ /dev/null
@@ -1,95 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
--module(emqx_connector_schema).
-
--behaviour(hocon_schema).
-
--include_lib("typerefl/include/types.hrl").
--include_lib("hocon/include/hoconsc.hrl").
-
--export([namespace/0, roots/0, fields/1, desc/1]).
-
--export([
-    get_response/0,
-    put_request/0,
-    post_request/0
-]).
-
-%% the config for webhook bridges do not need connectors
--define(CONN_TYPES, [mqtt]).
-
-%%======================================================================================
-%% For HTTP APIs
-
-get_response() ->
-    http_schema("get").
-
-put_request() ->
-    http_schema("put").
-
-post_request() ->
-    http_schema("post").
-
-http_schema(Method) ->
-    Broker = [?R_REF(schema_mod(Type), Method) || Type <- ?CONN_TYPES],
-    EE = ee_schemas(Method),
-    Schemas = Broker ++ EE,
-    ?UNION(Schemas).
-
-%%======================================================================================
-%% Hocon Schema Definitions
-
-namespace() -> connector.
-
-roots() -> ["connectors"].
-
-fields(connectors) ->
-    fields("connectors");
-fields("connectors") ->
-    Broker = [
-        {mqtt,
-            ?HOCON(
-                ?MAP(name, ?R_REF(emqx_connector_mqtt_schema, "connector")),
-                #{desc => ?DESC("mqtt")}
-            )}
-    ],
-    EE = ee_fields_connectors(),
-    Broker ++ EE.
-
--if(?EMQX_RELEASE_EDITION == ee).
-ee_schemas(Method) ->
-    emqx_ee_connector:api_schemas(Method).
-
-ee_fields_connectors() ->
-    emqx_ee_connector:fields(connectors).
--else.
-ee_fields_connectors() ->
-    [].
-
-ee_schemas(_) ->
-    [].
--endif.
-
-desc(Record) when
-    Record =:= connectors;
-    Record =:= "connectors"
-->
-    ?DESC("desc_connector");
-desc(_) ->
-    undefined.
-
-schema_mod(Type) ->
-    list_to_atom(lists:concat(["emqx_connector_", Type])).
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
index d0251104b..372405b59 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
@@ -207,7 +207,7 @@ make_hdlr(Parent, Vars, Opts) ->
 
 sub_remote_topics(_ClientPid, undefined) ->
     ok;
-sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) ->
+sub_remote_topics(ClientPid, #{remote := #{topic := FromTopic, qos := QoS}}) ->
     case emqtt:subscribe(ClientPid, FromTopic, QoS) of
         {ok, _, _} -> ok;
         Error -> throw(Error)
@@ -217,12 +217,10 @@ process_config(Config) ->
     maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config).
 
 maybe_publish_to_local_broker(Msg, Vars, Props) ->
-    case maps:get(local_topic, Vars, undefined) of
-        undefined ->
-            %% local topic is not set, discard it
-            ok;
-        _ ->
-            _ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars, Props))
+    case emqx_map_lib:deep_get([local, topic], Vars, undefined) of
+        %% local topic is not set, discard it
+        undefined -> ok;
+        _ -> emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars, Props))
     end.
 
 format_msg_received(
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
index 43700506b..7198521f2 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
@@ -38,14 +38,16 @@
 
 -type msg() :: emqx_types:message().
 -type exp_msg() :: emqx_types:message() | #mqtt_msg{}.
-
--type variables() :: #{
-    mountpoint := undefined | binary(),
-    remote_topic := binary(),
-    remote_qos := original | integer(),
+-type remote_config() :: #{
+    topic := binary(),
+    qos := original | integer(),
     retain := original | boolean(),
     payload := binary()
 }.
+-type variables() :: #{
+    mountpoint := undefined | binary(),
+    remote := remote_config()
+}.
 
 make_pub_vars(_, undefined) ->
     undefined;
@@ -67,10 +69,12 @@ to_remote_msg(#message{flags = Flags0} = Msg, Vars) ->
     MapMsg = maps:put(retain, Retain0, Columns),
     to_remote_msg(MapMsg, Vars);
 to_remote_msg(MapMsg, #{
-    remote_topic := TopicToken,
-    payload := PayloadToken,
-    remote_qos := QoSToken,
-    retain := RetainToken,
+    remote := #{
+        topic := TopicToken,
+        payload := PayloadToken,
+        qos := QoSToken,
+        retain := RetainToken
+    },
     mountpoint := Mountpoint
 }) when is_map(MapMsg) ->
     Topic = replace_vars_in_str(TopicToken, MapMsg),
@@ -91,11 +95,13 @@ to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) ->
 to_broker_msg(
     #{dup := Dup} = MapMsg,
     #{
-        local_topic := TopicToken,
-        payload := PayloadToken,
-        local_qos := QoSToken,
-        retain := RetainToken,
-        mountpoint := Mountpoint
+        local := #{
+            topic := TopicToken,
+            payload := PayloadToken,
+            qos := QoSToken,
+            retain := RetainToken,
+            mountpoint := Mountpoint
+        }
     },
     Props
 ) ->
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index 4d35583a3..040e5f392 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -28,25 +28,33 @@
     desc/1
 ]).
 
--export([
-    ingress_desc/0,
-    egress_desc/0
-]).
-
 -import(emqx_schema, [mk_duration/2]).
 
+-import(hoconsc, [mk/2, ref/2]).
+
 namespace() -> "connector-mqtt".
 
 roots() ->
     fields("config").
 
 fields("config") ->
-    fields("connector") ++
-        topic_mappings();
-fields("connector") ->
+    fields("server_configs") ++
+        [
+            {ingress,
+                mk(
+                    ref(?MODULE, "ingress"),
+                    #{default => #{}}
+                )},
+            {egress,
+                mk(
+                    ref(?MODULE, "egress"),
+                    #{default => #{}}
+                )}
+        ];
+fields("server_configs") ->
     [
         {mode,
-            sc(
+            mk(
                 hoconsc:enum([cluster_shareload]),
                 #{
                     default => cluster_shareload,
@@ -54,7 +62,7 @@ fields("connector") ->
                 }
             )},
         {server,
-            sc(
+            mk(
                 emqx_schema:ip_port(),
                 #{
                     required => true,
@@ -68,7 +76,7 @@ fields("connector") ->
                 #{default => "15s"}
             )},
         {proto_ver,
-            sc(
+            mk(
                 hoconsc:enum([v3, v4, v5]),
                 #{
                     default => v4,
@@ -76,7 +84,7 @@ fields("connector") ->
                 }
             )},
         {bridge_mode,
-            sc(
+            mk(
                 boolean(),
                 #{
                     default => false,
@@ -84,7 +92,7 @@ fields("connector") ->
                 }
             )},
         {username,
-            sc(
+            mk(
                 binary(),
                 #{
                     default => "emqx",
@@ -92,7 +100,7 @@ fields("connector") ->
                 }
             )},
         {password,
-            sc(
+            mk(
                 binary(),
                 #{
                     default => "emqx",
@@ -101,7 +109,7 @@ fields("connector") ->
                 }
             )},
         {clean_start,
-            sc(
+            mk(
                 boolean(),
                 #{
                     default => true,
@@ -116,20 +124,31 @@ fields("connector") ->
                 #{default => "15s"}
             )},
         {max_inflight,
-            sc(
+            mk(
                 non_neg_integer(),
                 #{
                     default => 32,
                     desc => ?DESC("max_inflight")
                 }
-            )},
-        {replayq, sc(ref("replayq"), #{})}
+            )}
     ] ++ emqx_connector_schema_lib:ssl_fields();
 fields("ingress") ->
-    %% the message maybe subscribed by rules, in this case 'local_topic' is not necessary
     [
-        {remote_topic,
-            sc(
+        {"remote",
+            mk(
+                ref(?MODULE, "ingress_remote"),
+                #{desc => ?DESC(emqx_connector_mqtt_schema, "ingress_remote")}
+            )},
+        {"local",
+            mk(
+                ref(?MODULE, "ingress_local"),
+                #{desc => ?DESC(emqx_connector_mqtt_schema, "ingress_local")}
+            )}
+    ];
+fields("ingress_remote") ->
+    [
+        {topic,
+            mk(
                 binary(),
                 #{
                     required => true,
@@ -137,47 +156,43 @@ fields("ingress") ->
                     desc => ?DESC("ingress_remote_topic")
                 }
             )},
-        {remote_qos,
-            sc(
+        {qos,
+            mk(
                 qos(),
                 #{
                     default => 1,
                     desc => ?DESC("ingress_remote_qos")
                 }
-            )},
-        {local_topic,
-            sc(
+            )}
+    ];
+fields("ingress_local") ->
+    [
+        {topic,
+            mk(
                 binary(),
                 #{
                     validator => fun emqx_schema:non_empty_string/1,
                     desc => ?DESC("ingress_local_topic")
                 }
             )},
-        {local_qos,
-            sc(
+        {qos,
+            mk(
                 qos(),
                 #{
                     default => <<"${qos}">>,
                     desc => ?DESC("ingress_local_qos")
                 }
             )},
-        {hookpoint,
-            sc(
-                binary(),
-                #{desc => ?DESC("ingress_hookpoint")}
-            )},
-
         {retain,
-            sc(
+            mk(
                 hoconsc:union([boolean(), binary()]),
                 #{
                     default => <<"${retain}">>,
                     desc => ?DESC("retain")
                 }
             )},
-
         {payload,
-            sc(
+            mk(
                 binary(),
                 #{
                     default => <<"${payload}">>,
@@ -186,18 +201,33 @@ fields("ingress") ->
             )}
     ];
 fields("egress") ->
-    %% the message maybe sent from rules, in this case 'local_topic' is not necessary
     [
-        {local_topic,
-            sc(
+        {"local",
+            mk(
+                ref(?MODULE, "egress_local"),
+                #{desc => ?DESC(emqx_connector_mqtt_schema, "egress_local")}
+            )},
+        {"remote",
+            mk(
+                ref(?MODULE, "egress_remote"),
+                #{desc => ?DESC(emqx_connector_mqtt_schema, "egress_remote")}
+            )}
+    ];
+fields("egress_local") ->
+    [
+        {topic,
+            mk(
                 binary(),
                 #{
                     desc => ?DESC("egress_local_topic"),
                     validator => fun emqx_schema:non_empty_string/1
                 }
-            )},
-        {remote_topic,
-            sc(
+            )}
+    ];
+fields("egress_remote") ->
+    [
+        {topic,
+            mk(
                 binary(),
                 #{
                     required => true,
@@ -205,104 +235,48 @@ fields("egress") ->
                     desc => ?DESC("egress_remote_topic")
                 }
             )},
-        {remote_qos,
-            sc(
+        {qos,
+            mk(
                 qos(),
                 #{
                     required => true,
                     desc => ?DESC("egress_remote_qos")
                 }
             )},
-
         {retain,
-            sc(
+            mk(
                 hoconsc:union([boolean(), binary()]),
                 #{
                     required => true,
                     desc => ?DESC("retain")
                 }
             )},
-
         {payload,
-            sc(
+            mk(
                 binary(),
                 #{
                     required => true,
                     desc => ?DESC("payload")
                 }
             )}
-    ];
-fields("replayq") ->
-    [
-        {dir,
-            sc(
-                hoconsc:union([boolean(), string()]),
-                #{desc => ?DESC("dir")}
-            )},
-        {seg_bytes,
-            sc(
-                emqx_schema:bytesize(),
-                #{
-                    default => "100MB",
-                    desc => ?DESC("seg_bytes")
-                }
-            )},
-        {offload,
-            sc(
-                boolean(),
-                #{
-                    default => false,
-                    desc => ?DESC("offload")
-                }
-            )}
     ].
 
-desc("connector") ->
-    ?DESC("desc_connector");
+desc("server_configs") ->
+    ?DESC("server_configs");
 desc("ingress") ->
-    ingress_desc();
+    ?DESC("ingress_desc");
+desc("ingress_remote") ->
+    ?DESC("ingress_remote");
+desc("ingress_local") ->
+    ?DESC("ingress_local");
 desc("egress") ->
-    egress_desc();
-desc("replayq") ->
-    ?DESC("desc_replayq");
+    ?DESC("egress_desc");
+desc("egress_remote") ->
+    ?DESC("egress_remote");
+desc("egress_local") ->
+    ?DESC("egress_local");
 desc(_) ->
     undefined.
 
-topic_mappings() ->
-    [
-        {ingress,
-            sc(
-                ref("ingress"),
-                #{default => #{}}
-            )},
-        {egress,
-            sc(
-                ref("egress"),
-                #{default => #{}}
-            )}
-    ].
-
-ingress_desc() ->
-    "\n"
-    "The ingress config defines how this bridge receive messages from the remote MQTT broker, and then\n"
-    "send them to the local broker.</br>\n"
-    "Template with variables is allowed in 'local_topic', 'remote_qos', 'qos', 'retain',\n"
-    "'payload'.</br>\n"
-    "NOTE: if this bridge is used as the input of a rule (emqx rule engine), and also local_topic is\n"
-    "configured, then messages got from the remote broker will be sent to both the 'local_topic' and\n"
-    "the rule.\n".
-
-egress_desc() ->
-    "\n"
-    "The egress config defines how this bridge forwards messages from the local broker to the remote\n"
-    "broker.</br>\n"
-    "Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.</br>\n"
-    "NOTE: if this bridge is used as the action of a rule (emqx rule engine), and also local_topic\n"
-    "is configured, then both the data got from the rule and the MQTT messages that matches\n"
-    "local_topic will be forwarded.\n".
-
 qos() ->
     hoconsc:union([emqx_schema:qos(), binary()]).
-
-sc(Type, Meta) -> hoconsc:mk(Type, Meta).
-ref(Field) -> hoconsc:ref(?MODULE, Field).
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index db795a4cf..af5f5d3e7 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -68,7 +68,6 @@
 %% APIs
 -export([
     start_link/1,
-    register_metrics/0,
     stop/1
 ]).
 
@@ -247,18 +246,19 @@ pre_process_opts(#{subscriptions := InConf, forwards := OutConf} = ConnectOpts)
 
 pre_process_in_out(_, undefined) ->
     undefined;
+pre_process_in_out(in, #{local := LC} = Conf) when is_map(Conf) ->
+    Conf#{local => pre_process_in_out_common(LC)};
 pre_process_in_out(in, Conf) when is_map(Conf) ->
-    Conf1 = pre_process_conf(local_topic, Conf),
-    Conf2 = pre_process_conf(local_qos, Conf1),
-    pre_process_in_out_common(Conf2);
-pre_process_in_out(out, Conf) when is_map(Conf) ->
-    Conf1 = pre_process_conf(remote_topic, Conf),
-    Conf2 = pre_process_conf(remote_qos, Conf1),
-    pre_process_in_out_common(Conf2).
+    %% have no 'local' field in the config
+    Conf;
+pre_process_in_out(out, #{remote := RC} = Conf) when is_map(Conf) ->
+    Conf#{remote => pre_process_in_out_common(RC)}.
 
-pre_process_in_out_common(Conf) ->
-    Conf1 = pre_process_conf(payload, Conf),
-    pre_process_conf(retain, Conf1).
+pre_process_in_out_common(Conf0) ->
+    Conf1 = pre_process_conf(topic, Conf0),
+    Conf2 = pre_process_conf(qos, Conf1),
+    Conf3 = pre_process_conf(payload, Conf2),
+    pre_process_conf(retain, Conf3).
 
 pre_process_conf(Key, Conf) ->
     case maps:find(Key, Conf) of
@@ -450,7 +450,6 @@ do_send(
 ) ->
     Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards),
     ExportMsg = fun(Message) ->
-        emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'),
         emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
     end,
     ?SLOG(debug, #{
@@ -551,15 +550,6 @@ format_mountpoint(Prefix) ->
 
 name(Id) -> list_to_atom(str(Id)).
 
-register_metrics() ->
-    lists:foreach(
-        fun emqx_metrics:ensure/1,
-        [
-            'bridge.mqtt.message_sent_to_remote',
-            'bridge.mqtt.message_received_from_remote'
-        ]
-    ).
-
 obfuscate(Map) ->
     maps:fold(
         fun(K, V, Acc) ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index ebc06d211..ad1dfaee8 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -7,7 +7,7 @@
 
 -export([
     api_schemas/1,
-    conn_bridge_examples/1,
+    examples/1,
     resource_type/1,
     fields/1
 ]).
@@ -28,7 +28,7 @@ schema_modules() ->
         emqx_ee_bridge_mysql
     ].
 
-conn_bridge_examples(Method) ->
+examples(Method) ->
     MergeFun =
         fun(Example, Examples) ->
             maps:merge(Examples, Example)
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
deleted file mode 100644
index 6846ea740..000000000
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl
+++ /dev/null
@@ -1,57 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
--module(emqx_ee_connector).
-
--import(hoconsc, [mk/2, enum/1, ref/2]).
-
--export([
-    fields/1,
-    connector_examples/1,
-    api_schemas/1
-]).
-
-api_schemas(Method) ->
-    [
-        ref(emqx_ee_connector_hstreamdb, Method),
-        ref(emqx_ee_connector_influxdb, "udp_" ++ Method),
-        ref(emqx_ee_connector_influxdb, "api_v1_" ++ Method),
-        ref(emqx_ee_connector_influxdb, "api_v2_" ++ Method)
-    ].
-
-fields(connectors) ->
-    [
-        {hstreamdb,
-            mk(
-                hoconsc:map(name, ref(emqx_ee_connector_hstreamdb, config)),
-                #{desc => <<"EMQX Enterprise Config">>}
-            )}
-    ] ++ fields(influxdb);
-fields(influxdb) ->
-    [
-        {
-            Protocol,
-            mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, Protocol)), #{
-                desc => <<"EMQX Enterprise Config">>
-            })
-        }
-     || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2]
-    ].
-
-connector_examples(Method) ->
-    MergeFun =
-        fun(Example, Examples) ->
-            maps:merge(Examples, Example)
-        end,
-    Fun =
-        fun(Module, Examples) ->
-            ConnectorExamples = erlang:apply(Module, connector_examples, [Method]),
-            lists:foldl(MergeFun, Examples, ConnectorExamples)
-        end,
-    lists:foldl(Fun, #{}, schema_modules()).
-
-schema_modules() ->
-    [
-        emqx_ee_connector_hstreamdb,
-        emqx_ee_connector_influxdb
-    ].

From 55c18c0b5f2bdacb752fe242493253de7e6e4edb Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 22 Aug 2022 16:05:47 +0800
Subject: [PATCH 086/232] fix(bridges): update the test cases for new config
 structure

---
 apps/emqx_bridge/test/emqx_bridge_SUITE.erl   |  36 +-
 .../test/emqx_bridge_mqtt_SUITE.erl           | 447 ++++++++++
 .../src/emqx_connector_mqtt.erl               |  16 +-
 .../src/mqtt/emqx_connector_mqtt_msg.erl      |   6 +-
 .../src/mqtt/emqx_connector_mqtt_schema.erl   |   8 +-
 .../src/mqtt/emqx_connector_mqtt_worker.erl   |   7 +-
 .../test/emqx_connector_SUITE.erl             |  94 --
 .../test/emqx_connector_api_SUITE.erl         | 812 ------------------
 8 files changed, 481 insertions(+), 945 deletions(-)
 create mode 100644 apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
 delete mode 100644 apps/emqx_connector/test/emqx_connector_SUITE.erl
 delete mode 100644 apps/emqx_connector/test/emqx_connector_api_SUITE.erl

diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
index d8266f83a..02abbddcb 100644
--- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
@@ -89,36 +89,29 @@ t_get_basic_usage_info_1(_Config) ->
     ).
 
 setup_fake_telemetry_data() ->
-    ConnectorConf =
-        #{
-            <<"connectors">> =>
-                #{
-                    <<"mqtt">> => #{
-                        <<"my_mqtt_connector">> =>
-                            #{server => "127.0.0.1:1883"},
-                        <<"my_mqtt_connector2">> =>
-                            #{server => "127.0.0.1:1884"}
-                    }
-                }
-        },
     MQTTConfig1 = #{
-        connector => <<"mqtt:my_mqtt_connector">>,
+        server => "127.0.0.1:1883",
         enable => true,
-        direction => ingress,
-        remote_topic => <<"aws/#">>,
-        remote_qos => 1
+        ingress => #{
+            remote => #{
+                topic => <<"aws/#">>,
+                qos => 1
+            }
+        }
     },
     MQTTConfig2 = #{
-        connector => <<"mqtt:my_mqtt_connector2">>,
+        server => "127.0.0.1:1884",
         enable => true,
-        direction => ingress,
-        remote_topic => <<"$bridges/mqtt:some_bridge_in">>,
-        remote_qos => 1
+        ingress => #{
+            remote => #{
+                topic => <<"$bridges/mqtt:some_bridge_in">>,
+                qos => 1
+            }
+        }
     },
     HTTPConfig = #{
         url => <<"http://localhost:9901/messages/${topic}">>,
         enable => true,
-        direction => egress,
         local_topic => "emqx_webhook/#",
         method => post,
         body => <<"${payload}">>,
@@ -143,7 +136,6 @@ setup_fake_telemetry_data() ->
                 }
         },
     Opts = #{raw_with_default => true},
-    ok = emqx_common_test_helpers:load_config(emqx_connector_schema, ConnectorConf, Opts),
     ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, Conf, Opts),
 
     ok = snabbkaffe:start_trace(),
diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
new file mode 100644
index 000000000..27c101728
--- /dev/null
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -0,0 +1,447 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_mqtt_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-import(emqx_dashboard_api_test_helpers, [request/4, uri/1]).
+
+-include("emqx/include/emqx.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include("emqx_dashboard/include/emqx_dashboard.hrl").
+
+%% output functions
+-export([inspect/3]).
+
+-define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
+-define(TYPE_MQTT, <<"mqtt">>).
+-define(NAME_MQTT, <<"my_mqtt_bridge">>).
+-define(BRIDGE_NAME_INGRESS, <<"ingress_mqtt_bridge">>).
+-define(BRIDGE_NAME_EGRESS, <<"egress_mqtt_bridge">>).
+-define(SERVER_CONF(Username), #{
+    <<"server">> => <<"127.0.0.1:1883">>,
+    <<"username">> => Username,
+    <<"password">> => <<"">>,
+    <<"proto_ver">> => <<"v4">>,
+    <<"ssl">> => #{<<"enable">> => false}
+}).
+
+-define(INGRESS_CONF, #{
+    <<"remote">> => #{
+        <<"topic">> => <<"remote_topic/#">>,
+        <<"qos">> => 2
+    },
+    <<"local">> => #{
+        <<"topic">> => <<"local_topic/${topic}">>,
+        <<"qos">> => <<"${qos}">>,
+        <<"payload">> => <<"${payload}">>,
+        <<"retain">> => <<"${retain}">>
+    }
+}).
+
+-define(EGRESS_CONF, #{
+    <<"local">> => #{
+        <<"topic">> => <<"local_topic/#">>
+    },
+    <<"remote">> => #{
+        <<"topic">> => <<"remote_topic/${topic}">>,
+        <<"payload">> => <<"${payload}">>,
+        <<"qos">> => <<"${qos}">>,
+        <<"retain">> => <<"${retain}">>
+    }
+}).
+
+-define(metrics(MATCH, SUCC, FAILED, SPEED, SPEED5M, SPEEDMAX), #{
+    <<"matched">> := MATCH,
+    <<"success">> := SUCC,
+    <<"failed">> := FAILED,
+    <<"rate">> := SPEED,
+    <<"rate_last5m">> := SPEED5M,
+    <<"rate_max">> := SPEEDMAX
+}).
+
+inspect(Selected, _Envs, _Args) ->
+    persistent_term:put(?MODULE, #{inspect => Selected}).
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+groups() ->
+    [].
+
+suite() ->
+    [{timetrap, {seconds, 30}}].
+
+init_per_suite(Config) ->
+    _ = application:load(emqx_conf),
+    %% some testcases (may from other app) already get emqx_connector started
+    _ = application:stop(emqx_resource),
+    _ = application:stop(emqx_connector),
+    ok = emqx_common_test_helpers:start_apps(
+        [
+            emqx_rule_engine,
+            emqx_bridge,
+            emqx_dashboard
+        ],
+        fun set_special_configs/1
+    ),
+    ok = emqx_common_test_helpers:load_config(
+        emqx_rule_engine_schema,
+        <<"rule_engine {rules {}}">>
+    ),
+    ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_common_test_helpers:stop_apps([
+        emqx_rule_engine,
+        emqx_bridge,
+        emqx_dashboard
+    ]),
+    ok.
+
+set_special_configs(emqx_dashboard) ->
+    emqx_dashboard_api_test_helpers:set_default_config(<<"connector_admin">>);
+set_special_configs(_) ->
+    ok.
+
+init_per_testcase(_, Config) ->
+    {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
+    Config.
+end_per_testcase(_, _Config) ->
+    clear_resources(),
+    ok.
+
+clear_resources() ->
+    lists:foreach(
+        fun(#{id := Id}) ->
+            ok = emqx_rule_engine:delete_rule(Id)
+        end,
+        emqx_rule_engine:get_rules()
+    ),
+    lists:foreach(
+        fun(#{type := Type, name := Name}) ->
+            {ok, _} = emqx_bridge:remove(Type, Name)
+        end,
+        emqx_bridge:list()
+    ).
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+t_mqtt_conn_bridge_ingress(_) ->
+    User1 = <<"user1">>,
+    %% create an MQTT bridge, using POST
+    {ok, 201, Bridge} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF(User1)#{
+            <<"type">> => ?TYPE_MQTT,
+            <<"name">> => ?BRIDGE_NAME_INGRESS,
+            <<"ingress">> => ?INGRESS_CONF
+        }
+    ),
+    #{
+        <<"type">> := ?TYPE_MQTT,
+        <<"name">> := ?BRIDGE_NAME_INGRESS
+    } = jsx:decode(Bridge),
+    BridgeIDIngress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_INGRESS),
+
+    %% we now test if the bridge works as expected
+    RemoteTopic = <<"remote_topic/1">>,
+    LocalTopic = <<"local_topic/", RemoteTopic/binary>>,
+    Payload = <<"hello">>,
+    emqx:subscribe(LocalTopic),
+    timer:sleep(100),
+    %% PUBLISH a message to the 'remote' broker, as we have only one broker,
+    %% the remote broker is also the local one.
+    emqx:publish(emqx_message:make(RemoteTopic, Payload)),
+    %% we should receive a message on the local broker, with specified topic
+    ?assert(
+        receive
+            {deliver, LocalTopic, #message{payload = Payload}} ->
+                ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end
+    ),
+
+    %% delete the bridge
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+
+    ok.
+
+t_mqtt_conn_bridge_egress(_) ->
+    %% then we add a mqtt connector, using POST
+    User1 = <<"user1">>,
+
+    {ok, 201, Bridge} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF(User1)#{
+            <<"type">> => ?TYPE_MQTT,
+            <<"name">> => ?BRIDGE_NAME_EGRESS,
+            <<"egress">> => ?EGRESS_CONF
+        }
+    ),
+    #{
+        <<"type">> := ?TYPE_MQTT,
+        <<"name">> := ?BRIDGE_NAME_EGRESS
+    } = jsx:decode(Bridge),
+    BridgeIDEgress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_EGRESS),
+    %% we now test if the bridge works as expected
+    LocalTopic = <<"local_topic/1">>,
+    RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
+    Payload = <<"hello">>,
+    emqx:subscribe(RemoteTopic),
+    timer:sleep(100),
+    %% PUBLISH a message to the 'local' broker, as we have only one broker,
+    %% the remote broker is also the local one.
+    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+
+    %% we should receive a message on the "remote" broker, with specified topic
+    ?assert(
+        receive
+            {deliver, RemoteTopic, #message{payload = Payload}} ->
+                ct:pal("local broker got message: ~p on topic ~p", [Payload, RemoteTopic]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end
+    ),
+
+    %% verify the metrics of the bridge
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(
+        #{
+            <<"metrics">> := ?metrics(1, 1, 0, _, _, _),
+            <<"node_metrics">> :=
+                [#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}]
+        },
+        jsx:decode(BridgeStr)
+    ),
+
+    %% delete the bridge
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+    ok.
+
+t_ingress_mqtt_bridge_with_rules(_) ->
+    {ok, 201, _} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF(<<"user1">>)#{
+            <<"type">> => ?TYPE_MQTT,
+            <<"name">> => ?BRIDGE_NAME_INGRESS,
+            <<"ingress">> => ?INGRESS_CONF
+        }
+    ),
+    BridgeIDIngress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_INGRESS),
+
+    {ok, 201, Rule} = request(
+        post,
+        uri(["rules"]),
+        #{
+            <<"name">> => <<"A_rule_get_messages_from_a_source_mqtt_bridge">>,
+            <<"enable">> => true,
+            <<"actions">> => [#{<<"function">> => "emqx_bridge_mqtt_SUITE:inspect"}],
+            <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">>
+        }
+    ),
+    #{<<"id">> := RuleId} = jsx:decode(Rule),
+
+    %% we now test if the bridge works as expected
+
+    RemoteTopic = <<"remote_topic/1">>,
+    LocalTopic = <<"local_topic/", RemoteTopic/binary>>,
+    Payload = <<"hello">>,
+    emqx:subscribe(LocalTopic),
+    timer:sleep(100),
+    %% PUBLISH a message to the 'remote' broker, as we have only one broker,
+    %% the remote broker is also the local one.
+    emqx:publish(emqx_message:make(RemoteTopic, Payload)),
+    %% we should receive a message on the local broker, with specified topic
+    ?assert(
+        receive
+            {deliver, LocalTopic, #message{payload = Payload}} ->
+                ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end
+    ),
+    %% and also the rule should be matched, with matched + 1:
+    {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
+    ?assertMatch(
+        #{
+            <<"id">> := RuleId,
+            <<"metrics">> := #{
+                <<"matched">> := 1,
+                <<"passed">> := 1,
+                <<"failed">> := 0,
+                <<"failed.exception">> := 0,
+                <<"failed.no_result">> := 0,
+                <<"matched.rate">> := _,
+                <<"matched.rate.max">> := _,
+                <<"matched.rate.last5m">> := _,
+                <<"actions.total">> := 1,
+                <<"actions.success">> := 1,
+                <<"actions.failed">> := 0,
+                <<"actions.failed.out_of_service">> := 0,
+                <<"actions.failed.unknown">> := 0
+            }
+        },
+        jsx:decode(Rule1)
+    ),
+    %% we also check if the actions of the rule is triggered
+    ?assertMatch(
+        #{
+            inspect := #{
+                event := <<"$bridges/mqtt", _/binary>>,
+                id := MsgId,
+                payload := Payload,
+                topic := RemoteTopic,
+                qos := 0,
+                dup := false,
+                retain := false,
+                pub_props := #{},
+                timestamp := _
+            }
+        } when is_binary(MsgId),
+        persistent_term:get(?MODULE)
+    ),
+
+    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []).
+
+t_egress_mqtt_bridge_with_rules(_) ->
+    {ok, 201, Bridge} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF(<<"user1">>)#{
+            <<"type">> => ?TYPE_MQTT,
+            <<"name">> => ?BRIDGE_NAME_EGRESS,
+            <<"egress">> => ?EGRESS_CONF
+        }
+    ),
+    #{<<"type">> := ?TYPE_MQTT, <<"name">> := ?BRIDGE_NAME_EGRESS} = jsx:decode(Bridge),
+    BridgeIDEgress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_EGRESS),
+
+    {ok, 201, Rule} = request(
+        post,
+        uri(["rules"]),
+        #{
+            <<"name">> => <<"A_rule_send_messages_to_a_sink_mqtt_bridge">>,
+            <<"enable">> => true,
+            <<"actions">> => [BridgeIDEgress],
+            <<"sql">> => <<"SELECT * from \"t/1\"">>
+        }
+    ),
+    #{<<"id">> := RuleId} = jsx:decode(Rule),
+
+    %% we now test if the bridge works as expected
+    LocalTopic = <<"local_topic/1">>,
+    RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
+    Payload = <<"hello">>,
+    emqx:subscribe(RemoteTopic),
+    timer:sleep(100),
+    %% PUBLISH a message to the 'local' broker, as we have only one broker,
+    %% the remote broker is also the local one.
+    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+    %% we should receive a message on the "remote" broker, with specified topic
+    ?assert(
+        receive
+            {deliver, RemoteTopic, #message{payload = Payload}} ->
+                ct:pal("remote broker got message: ~p on topic ~p", [Payload, RemoteTopic]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end
+    ),
+    emqx:unsubscribe(RemoteTopic),
+
+    %% PUBLISH a message to the rule.
+    Payload2 = <<"hi">>,
+    RuleTopic = <<"t/1">>,
+    RemoteTopic2 = <<"remote_topic/", RuleTopic/binary>>,
+    emqx:subscribe(RemoteTopic2),
+    timer:sleep(100),
+    emqx:publish(emqx_message:make(RuleTopic, Payload2)),
+    {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
+    #{
+        <<"id">> := RuleId,
+        <<"metrics">> := #{
+            <<"matched">> := 1,
+            <<"passed">> := 1,
+            <<"failed">> := 0,
+            <<"failed.exception">> := 0,
+            <<"failed.no_result">> := 0,
+            <<"matched.rate">> := _,
+            <<"matched.rate.max">> := _,
+            <<"matched.rate.last5m">> := _,
+            <<"actions.total">> := 1,
+            <<"actions.success">> := 1,
+            <<"actions.failed">> := 0,
+            <<"actions.failed.out_of_service">> := 0,
+            <<"actions.failed.unknown">> := 0
+        }
+    } = jsx:decode(Rule1),
+    %% we should receive a message on the "remote" broker, with specified topic
+    ?assert(
+        receive
+            {deliver, RemoteTopic2, #message{payload = Payload2}} ->
+                ct:pal("remote broker got message: ~p on topic ~p", [Payload2, RemoteTopic2]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end
+    ),
+
+    %% verify the metrics of the bridge
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(
+        #{
+            <<"metrics">> := ?metrics(2, 2, 0, _, _, _),
+            <<"node_metrics">> :=
+                [#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}]
+        },
+        jsx:decode(BridgeStr)
+    ),
+
+    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []).
+
+request(Method, Url, Body) ->
+    request(<<"connector_admin">>, Method, Url, Body).
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index b57a94b36..bdf43885a 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -153,7 +153,7 @@ on_start(InstId, Conf) ->
     BridgeConf = BasicConf#{
         name => InstanceId,
         clientid => clientid(InstId),
-        subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), InstId),
+        subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), Conf, InstId),
         forwards => make_forward_confs(maps:get(egress, Conf, undefined))
     },
     case ?MODULE:create_bridge(BridgeConf) of
@@ -204,18 +204,18 @@ ensure_mqtt_worker_started(InstanceId, BridgeConf) ->
         {error, Reason} -> {error, Reason}
     end.
 
-make_sub_confs(EmptyMap, _) when map_size(EmptyMap) == 0 ->
+make_sub_confs(EmptyMap, _Conf, _) when map_size(EmptyMap) == 0 ->
     undefined;
-make_sub_confs(undefined, _) ->
+make_sub_confs(undefined, _Conf, _) ->
     undefined;
-make_sub_confs(SubRemoteConf, InstId) ->
+make_sub_confs(SubRemoteConf, Conf, InstId) ->
     ResId = emqx_resource_manager:manager_id_to_resource_id(InstId),
-    case maps:take(hookpoint, SubRemoteConf) of
+    case maps:find(hookpoint, Conf) of
         error ->
-            SubRemoteConf;
-        {HookPoint, SubConf} ->
+            error({no_hookpoint_provided, Conf});
+        {ok, HookPoint} ->
             MFA = {?MODULE, on_message_received, [HookPoint, ResId]},
-            SubConf#{on_message_received => MFA}
+            SubRemoteConf#{on_message_received => MFA}
     end.
 
 make_forward_confs(EmptyMap) when map_size(EmptyMap) == 0 ->
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
index 7198521f2..fd5fc04f9 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
@@ -99,9 +99,9 @@ to_broker_msg(
             topic := TopicToken,
             payload := PayloadToken,
             qos := QoSToken,
-            retain := RetainToken,
-            mountpoint := Mountpoint
-        }
+            retain := RetainToken
+        },
+        mountpoint := Mountpoint
     },
     Props
 ) ->
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index 040e5f392..d6c6b1fb7 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -42,13 +42,13 @@ fields("config") ->
         [
             {ingress,
                 mk(
-                    ref(?MODULE, "ingress"),
-                    #{default => #{}}
+                    hoconsc:union([none, ref(?MODULE, "ingress")]),
+                    #{default => undefined}
                 )},
             {egress,
                 mk(
-                    ref(?MODULE, "egress"),
-                    #{default => #{}}
+                    hoconsc:union([none, ref(?MODULE, "egress")]),
+                    #{default => undefined}
                 )}
         ];
 fields("server_configs") ->
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index af5f5d3e7..3f3a4b9ce 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -250,9 +250,12 @@ pre_process_in_out(in, #{local := LC} = Conf) when is_map(Conf) ->
     Conf#{local => pre_process_in_out_common(LC)};
 pre_process_in_out(in, Conf) when is_map(Conf) ->
     %% have no 'local' field in the config
-    Conf;
+    undefined;
 pre_process_in_out(out, #{remote := RC} = Conf) when is_map(Conf) ->
-    Conf#{remote => pre_process_in_out_common(RC)}.
+    Conf#{remote => pre_process_in_out_common(RC)};
+pre_process_in_out(out, Conf) when is_map(Conf) ->
+    %% have no 'remote' field in the config
+    undefined.
 
 pre_process_in_out_common(Conf0) ->
     Conf1 = pre_process_conf(topic, Conf0),
diff --git a/apps/emqx_connector/test/emqx_connector_SUITE.erl b/apps/emqx_connector/test/emqx_connector_SUITE.erl
deleted file mode 100644
index c4a6418c2..000000000
--- a/apps/emqx_connector/test/emqx_connector_SUITE.erl
+++ /dev/null
@@ -1,94 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_connector_SUITE).
-
--compile(nowarn_export_all).
--compile(export_all).
-
--include("emqx/include/emqx.hrl").
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
-
--define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
--define(MQTT_CONNECTOR(Username), #{
-    <<"server">> => <<"127.0.0.1:1883">>,
-    <<"username">> => Username,
-    <<"password">> => <<"">>,
-    <<"proto_ver">> => <<"v4">>,
-    <<"ssl">> => #{<<"enable">> => false}
-}).
--define(CONNECTOR_TYPE, <<"mqtt">>).
--define(CONNECTOR_NAME, <<"test_connector_42">>).
-
-all() ->
-    emqx_common_test_helpers:all(?MODULE).
-
-groups() ->
-    [].
-
-suite() ->
-    [].
-
-init_per_suite(Config) ->
-    _ = application:load(emqx_conf),
-    %% some testcases (may from other app) already get emqx_connector started
-    _ = application:stop(emqx_resource),
-    _ = application:stop(emqx_connector),
-    ok = emqx_common_test_helpers:start_apps(
-        [
-            emqx_connector,
-            emqx_bridge
-        ]
-    ),
-    ok = emqx_common_test_helpers:load_config(emqx_connector_schema, <<"connectors: {}">>),
-    Config.
-
-end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([
-        emqx_connector,
-        emqx_bridge
-    ]),
-    ok.
-
-init_per_testcase(_, Config) ->
-    {ok, _} = emqx_cluster_rpc:start_link(),
-    Config.
-end_per_testcase(_, _Config) ->
-    ok.
-
-t_list_raw_empty(_) ->
-    ok = emqx_config:erase(hd(emqx_connector:config_key_path())),
-    Result = emqx_connector:list_raw(),
-    ?assertEqual([], Result).
-
-t_lookup_raw_error(_) ->
-    Result = emqx_connector:lookup_raw(<<"foo:bar">>),
-    ?assertEqual({error, not_found}, Result).
-
-t_parse_connector_id_error(_) ->
-    ?assertError(
-        {invalid_connector_id, <<"foobar">>}, emqx_connector:parse_connector_id(<<"foobar">>)
-    ).
-
-t_update_connector_does_not_exist(_) ->
-    Config = ?MQTT_CONNECTOR(<<"user1">>),
-    ?assertMatch({ok, _Config}, emqx_connector:update(?CONNECTOR_TYPE, ?CONNECTOR_NAME, Config)).
-
-t_delete_connector_does_not_exist(_) ->
-    ?assertEqual({ok, #{post_config_update => #{}}}, emqx_connector:delete(<<"foo:bar">>)).
-
-t_connector_id_using_list(_) ->
-    <<"foo:bar">> = emqx_connector:connector_id("foo", "bar").
diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl
deleted file mode 100644
index 8c2405fde..000000000
--- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl
+++ /dev/null
@@ -1,812 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_connector_api_SUITE).
-
--compile(nowarn_export_all).
--compile(export_all).
-
--import(emqx_dashboard_api_test_helpers, [request/4, uri/1]).
-
--include("emqx/include/emqx.hrl").
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include("emqx_dashboard/include/emqx_dashboard.hrl").
-
-%% output functions
--export([inspect/3]).
-
--define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
--define(CONNECTR_TYPE, <<"mqtt">>).
--define(CONNECTR_NAME, <<"test_connector">>).
--define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>).
--define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>).
--define(MQTT_CONNECTOR(Username), #{
-    <<"server">> => <<"127.0.0.1:1883">>,
-    <<"username">> => Username,
-    <<"password">> => <<"">>,
-    <<"proto_ver">> => <<"v4">>,
-    <<"ssl">> => #{<<"enable">> => false}
-}).
--define(MQTT_CONNECTOR2(Server), ?MQTT_CONNECTOR(<<"user1">>)#{<<"server">> => Server}).
-
--define(MQTT_BRIDGE_INGRESS(ID), #{
-    <<"connector">> => ID,
-    <<"direction">> => <<"ingress">>,
-    <<"remote_topic">> => <<"remote_topic/#">>,
-    <<"remote_qos">> => 2,
-    <<"local_topic">> => <<"local_topic/${topic}">>,
-    <<"local_qos">> => <<"${qos}">>,
-    <<"payload">> => <<"${payload}">>,
-    <<"retain">> => <<"${retain}">>
-}).
-
--define(MQTT_BRIDGE_EGRESS(ID), #{
-    <<"connector">> => ID,
-    <<"direction">> => <<"egress">>,
-    <<"local_topic">> => <<"local_topic/#">>,
-    <<"remote_topic">> => <<"remote_topic/${topic}">>,
-    <<"payload">> => <<"${payload}">>,
-    <<"remote_qos">> => <<"${qos}">>,
-    <<"retain">> => <<"${retain}">>
-}).
-
--define(metrics(MATCH, SUCC, FAILED, SPEED, SPEED5M, SPEEDMAX), #{
-    <<"matched">> := MATCH,
-    <<"success">> := SUCC,
-    <<"failed">> := FAILED,
-    <<"rate">> := SPEED,
-    <<"rate_last5m">> := SPEED5M,
-    <<"rate_max">> := SPEEDMAX
-}).
-
-inspect(Selected, _Envs, _Args) ->
-    persistent_term:put(?MODULE, #{inspect => Selected}).
-
-all() ->
-    emqx_common_test_helpers:all(?MODULE).
-
-groups() ->
-    [].
-
-suite() ->
-    [{timetrap, {seconds, 30}}].
-
-init_per_suite(Config) ->
-    _ = application:load(emqx_conf),
-    %% some testcases (may from other app) already get emqx_connector started
-    _ = application:stop(emqx_resource),
-    _ = application:stop(emqx_connector),
-    ok = emqx_common_test_helpers:start_apps(
-        [
-            emqx_rule_engine,
-            emqx_connector,
-            emqx_bridge,
-            emqx_dashboard
-        ],
-        fun set_special_configs/1
-    ),
-    ok = emqx_common_test_helpers:load_config(emqx_connector_schema, <<"connectors: {}">>),
-    ok = emqx_common_test_helpers:load_config(
-        emqx_rule_engine_schema,
-        <<"rule_engine {rules {}}">>
-    ),
-    ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
-    Config.
-
-end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([
-        emqx_rule_engine,
-        emqx_connector,
-        emqx_bridge,
-        emqx_dashboard
-    ]),
-    ok.
-
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config(<<"connector_admin">>);
-set_special_configs(_) ->
-    ok.
-
-init_per_testcase(_, Config) ->
-    {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
-    Config.
-end_per_testcase(_, _Config) ->
-    clear_resources(),
-    ok.
-
-clear_resources() ->
-    lists:foreach(
-        fun(#{id := Id}) ->
-            ok = emqx_rule_engine:delete_rule(Id)
-        end,
-        emqx_rule_engine:get_rules()
-    ),
-    lists:foreach(
-        fun(#{type := Type, name := Name}) ->
-            {ok, _} = emqx_bridge:remove(Type, Name)
-        end,
-        emqx_bridge:list()
-    ),
-    lists:foreach(
-        fun(#{<<"type">> := Type, <<"name">> := Name}) ->
-            {ok, _} = emqx_connector:delete(Type, Name)
-        end,
-        emqx_connector:list_raw()
-    ).
-
-%%------------------------------------------------------------------------------
-%% Testcases
-%%------------------------------------------------------------------------------
-
-t_mqtt_crud_apis(_) ->
-    %% assert we there's no connectors at first
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-
-    %% then we add a mqtt connector, using POST
-    %% POST /connectors/ will create a connector
-    User1 = <<"user1">>,
-    {ok, 400, <<
-        "{\"code\":\"BAD_REQUEST\",\"message\""
-        ":\"missing some required fields: [name, type]\"}"
-    >>} =
-        request(
-            post,
-            uri(["connectors"]),
-            ?MQTT_CONNECTOR(User1)#{<<"type">> => ?CONNECTR_TYPE}
-        ),
-    {ok, 201, Connector} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR(User1)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-
-    #{
-        <<"type">> := ?CONNECTR_TYPE,
-        <<"name">> := ?CONNECTR_NAME,
-        <<"server">> := <<"127.0.0.1:1883">>,
-        <<"username">> := User1,
-        <<"password">> := <<"">>,
-        <<"proto_ver">> := <<"v4">>,
-        <<"ssl">> := #{<<"enable">> := false}
-    } = jsx:decode(Connector),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-    %% update the request-path of the connector
-    User2 = <<"user2">>,
-    {ok, 200, Connector2} = request(
-        put,
-        uri(["connectors", ConnctorID]),
-        ?MQTT_CONNECTOR(User2)
-    ),
-    ?assertMatch(
-        #{
-            <<"type">> := ?CONNECTR_TYPE,
-            <<"name">> := ?CONNECTR_NAME,
-            <<"server">> := <<"127.0.0.1:1883">>,
-            <<"username">> := User2,
-            <<"password">> := <<"">>,
-            <<"proto_ver">> := <<"v4">>,
-            <<"ssl">> := #{<<"enable">> := false}
-        },
-        jsx:decode(Connector2)
-    ),
-
-    %% list all connectors again, assert Connector2 is in it
-    {ok, 200, Connector2Str} = request(get, uri(["connectors"]), []),
-    ?assertMatch(
-        [
-            #{
-                <<"type">> := ?CONNECTR_TYPE,
-                <<"name">> := ?CONNECTR_NAME,
-                <<"server">> := <<"127.0.0.1:1883">>,
-                <<"username">> := User2,
-                <<"password">> := <<"">>,
-                <<"proto_ver">> := <<"v4">>,
-                <<"ssl">> := #{<<"enable">> := false}
-            }
-        ],
-        jsx:decode(Connector2Str)
-    ),
-
-    %% get the connector by id
-    {ok, 200, Connector3Str} = request(get, uri(["connectors", ConnctorID]), []),
-    ?assertMatch(
-        #{
-            <<"type">> := ?CONNECTR_TYPE,
-            <<"name">> := ?CONNECTR_NAME,
-            <<"server">> := <<"127.0.0.1:1883">>,
-            <<"username">> := User2,
-            <<"password">> := <<"">>,
-            <<"proto_ver">> := <<"v4">>,
-            <<"ssl">> := #{<<"enable">> := false}
-        },
-        jsx:decode(Connector3Str)
-    ),
-
-    %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-
-    %% update a deleted connector returns an error
-    {ok, 404, ErrMsg2} = request(
-        put,
-        uri(["connectors", ConnctorID]),
-        ?MQTT_CONNECTOR(User2)
-    ),
-    ?assertMatch(
-        #{
-            <<"code">> := _,
-            <<"message">> := <<"connector not found">>
-        },
-        jsx:decode(ErrMsg2)
-    ),
-    ok.
-
-t_mqtt_conn_bridge_ingress(_) ->
-    %% then we add a mqtt connector, using POST
-    User1 = <<"user1">>,
-    {ok, 201, Connector} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR(User1)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-
-    #{
-        <<"type">> := ?CONNECTR_TYPE,
-        <<"name">> := ?CONNECTR_NAME,
-        <<"server">> := <<"127.0.0.1:1883">>,
-        <<"num_of_bridges">> := 0,
-        <<"username">> := User1,
-        <<"password">> := <<"">>,
-        <<"proto_ver">> := <<"v4">>,
-        <<"ssl">> := #{<<"enable">> := false}
-    } = jsx:decode(Connector),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-    %% ... and a MQTT bridge, using POST
-    %% we bind this bridge to the connector created just now
-    timer:sleep(50),
-    {ok, 201, Bridge} = request(
-        post,
-        uri(["bridges"]),
-        ?MQTT_BRIDGE_INGRESS(ConnctorID)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_INGRESS
-        }
-    ),
-    #{
-        <<"type">> := ?CONNECTR_TYPE,
-        <<"name">> := ?BRIDGE_NAME_INGRESS,
-        <<"connector">> := ConnctorID
-    } = jsx:decode(Bridge),
-    BridgeIDIngress = emqx_bridge_resource:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_INGRESS),
-    wait_for_resource_ready(BridgeIDIngress, 5),
-
-    %% we now test if the bridge works as expected
-    RemoteTopic = <<"remote_topic/1">>,
-    LocalTopic = <<"local_topic/", RemoteTopic/binary>>,
-    Payload = <<"hello">>,
-    emqx:subscribe(LocalTopic),
-    timer:sleep(100),
-    %% PUBLISH a message to the 'remote' broker, as we have only one broker,
-    %% the remote broker is also the local one.
-    emqx:publish(emqx_message:make(RemoteTopic, Payload)),
-    %% we should receive a message on the local broker, with specified topic
-    ?assert(
-        receive
-            {deliver, LocalTopic, #message{payload = Payload}} ->
-                ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]),
-                true;
-            Msg ->
-                ct:pal("Msg: ~p", [Msg]),
-                false
-        after 100 ->
-            false
-        end
-    ),
-
-    %% get the connector by id, verify the num_of_bridges now is 1
-    {ok, 200, Connector1Str} = request(get, uri(["connectors", ConnctorID]), []),
-    ?assertMatch(#{<<"num_of_bridges">> := 1}, jsx:decode(Connector1Str)),
-
-    %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
-    %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-    ok.
-
-t_mqtt_conn_bridge_egress(_) ->
-    %% then we add a mqtt connector, using POST
-    User1 = <<"user1">>,
-    {ok, 201, Connector} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR(User1)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-
-    %ct:pal("---connector: ~p", [Connector]),
-    #{
-        <<"server">> := <<"127.0.0.1:1883">>,
-        <<"username">> := User1,
-        <<"password">> := <<"">>,
-        <<"proto_ver">> := <<"v4">>,
-        <<"ssl">> := #{<<"enable">> := false}
-    } = jsx:decode(Connector),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-    %% ... and a MQTT bridge, using POST
-    %% we bind this bridge to the connector created just now
-    {ok, 201, Bridge} = request(
-        post,
-        uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_EGRESS
-        }
-    ),
-    #{
-        <<"type">> := ?CONNECTR_TYPE,
-        <<"name">> := ?BRIDGE_NAME_EGRESS,
-        <<"connector">> := ConnctorID
-    } = jsx:decode(Bridge),
-    BridgeIDEgress = emqx_bridge_resource:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
-    wait_for_resource_ready(BridgeIDEgress, 5),
-
-    %% we now test if the bridge works as expected
-    LocalTopic = <<"local_topic/1">>,
-    RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
-    Payload = <<"hello">>,
-    emqx:subscribe(RemoteTopic),
-    timer:sleep(100),
-    %% PUBLISH a message to the 'local' broker, as we have only one broker,
-    %% the remote broker is also the local one.
-    emqx:publish(emqx_message:make(LocalTopic, Payload)),
-
-    %% we should receive a message on the "remote" broker, with specified topic
-    ?assert(
-        receive
-            {deliver, RemoteTopic, #message{payload = Payload}} ->
-                ct:pal("local broker got message: ~p on topic ~p", [Payload, RemoteTopic]),
-                true;
-            Msg ->
-                ct:pal("Msg: ~p", [Msg]),
-                false
-        after 100 ->
-            false
-        end
-    ),
-
-    %% verify the metrics of the bridge
-    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
-    ?assertMatch(
-        #{
-            <<"metrics">> := ?metrics(1, 1, 0, _, _, _),
-            <<"node_metrics">> :=
-                [#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}]
-        },
-        jsx:decode(BridgeStr)
-    ),
-
-    %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
-    %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-    ok.
-
-%% t_mqtt_conn_update:
-%% - update a connector should also update all of the the bridges
-%% - cannot delete a connector that is used by at least one bridge
-t_mqtt_conn_update(_) ->
-    %% then we add a mqtt connector, using POST
-    {ok, 201, Connector} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-
-    %ct:pal("---connector: ~p", [Connector]),
-    #{<<"server">> := <<"127.0.0.1:1883">>} = jsx:decode(Connector),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-    %% ... and a MQTT bridge, using POST
-    %% we bind this bridge to the connector created just now
-    {ok, 201, Bridge} = request(
-        post,
-        uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_EGRESS
-        }
-    ),
-    #{
-        <<"type">> := ?CONNECTR_TYPE,
-        <<"name">> := ?BRIDGE_NAME_EGRESS,
-        <<"connector">> := ConnctorID
-    } = jsx:decode(Bridge),
-    BridgeIDEgress = emqx_bridge_resource:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
-    wait_for_resource_ready(BridgeIDEgress, 5),
-
-    %% Then we try to update 'server' of the connector, to an unavailable IP address
-    %% The update OK, we recreate the resource even if the resource is current connected,
-    %% and the target resource we're going to update is unavailable.
-    {ok, 200, _} = request(
-        put,
-        uri(["connectors", ConnctorID]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:2603">>)
-    ),
-    %% we fix the 'server' parameter to a normal one, it should work
-    {ok, 200, _} = request(
-        put,
-        uri(["connectors", ConnctorID]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1 : 1883">>)
-    ),
-    %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
-    %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
-
-t_mqtt_conn_update2(_) ->
-    %% then we add a mqtt connector, using POST
-    %% but this connector is point to a unreachable server "2603"
-    {ok, 201, Connector} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:2603">>)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-
-    #{<<"server">> := <<"127.0.0.1:2603">>} = jsx:decode(Connector),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-    %% ... and a MQTT bridge, using POST
-    %% we bind this bridge to the connector created just now
-    {ok, 201, Bridge} = request(
-        post,
-        uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_EGRESS
-        }
-    ),
-    #{
-        <<"type">> := ?CONNECTR_TYPE,
-        <<"name">> := ?BRIDGE_NAME_EGRESS,
-        <<"status">> := <<"disconnected">>,
-        <<"connector">> := ConnctorID
-    } = jsx:decode(Bridge),
-    BridgeIDEgress = emqx_bridge_resource:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
-    %% We try to fix the 'server' parameter, to another unavailable server..
-    %% The update should success: we don't check the connectivity of the new config
-    %% if the resource is now disconnected.
-    {ok, 200, _} = request(
-        put,
-        uri(["connectors", ConnctorID]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:2604">>)
-    ),
-    %% we fix the 'server' parameter to a normal one, it should work
-    {ok, 200, _} = request(
-        put,
-        uri(["connectors", ConnctorID]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)
-    ),
-    wait_for_resource_ready(BridgeIDEgress, 5),
-    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
-    ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(BridgeStr)),
-    %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
-    %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
-
-t_mqtt_conn_update3(_) ->
-    %% we add a mqtt connector, using POST
-    {ok, 201, _} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-    %% ... and a MQTT bridge, using POST
-    %% we bind this bridge to the connector created just now
-    {ok, 201, Bridge} = request(
-        post,
-        uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_EGRESS
-        }
-    ),
-    #{<<"connector">> := ConnctorID} = jsx:decode(Bridge),
-    BridgeIDEgress = emqx_bridge_resource:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
-    wait_for_resource_ready(BridgeIDEgress, 5),
-
-    %% delete the connector should fail because it is in use by a bridge
-    {ok, 403, _} = request(delete, uri(["connectors", ConnctorID]), []),
-    %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
-    %% the connector now can be deleted without problems
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
-
-t_mqtt_conn_testing(_) ->
-    %% APIs for testing the connectivity
-    %% then we add a mqtt connector, using POST
-    {ok, 204, <<>>} = request(
-        post,
-        uri(["connectors_test"]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_EGRESS
-        }
-    ),
-    {ok, 400, _} = request(
-        post,
-        uri(["connectors_test"]),
-        ?MQTT_CONNECTOR2(<<"127.0.0.1:2883">>)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_EGRESS
-        }
-    ).
-
-t_ingress_mqtt_bridge_with_rules(_) ->
-    {ok, 201, _} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR(<<"user1">>)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-
-    {ok, 201, _} = request(
-        post,
-        uri(["bridges"]),
-        ?MQTT_BRIDGE_INGRESS(ConnctorID)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_INGRESS
-        }
-    ),
-    BridgeIDIngress = emqx_bridge_resource:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_INGRESS),
-
-    {ok, 201, Rule} = request(
-        post,
-        uri(["rules"]),
-        #{
-            <<"name">> => <<"A_rule_get_messages_from_a_source_mqtt_bridge">>,
-            <<"enable">> => true,
-            <<"actions">> => [#{<<"function">> => "emqx_connector_api_SUITE:inspect"}],
-            <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">>
-        }
-    ),
-    #{<<"id">> := RuleId} = jsx:decode(Rule),
-
-    %% we now test if the bridge works as expected
-
-    RemoteTopic = <<"remote_topic/1">>,
-    LocalTopic = <<"local_topic/", RemoteTopic/binary>>,
-    Payload = <<"hello">>,
-    emqx:subscribe(LocalTopic),
-    timer:sleep(100),
-    %% PUBLISH a message to the 'remote' broker, as we have only one broker,
-    %% the remote broker is also the local one.
-    wait_for_resource_ready(BridgeIDIngress, 5),
-    emqx:publish(emqx_message:make(RemoteTopic, Payload)),
-    %% we should receive a message on the local broker, with specified topic
-    ?assert(
-        receive
-            {deliver, LocalTopic, #message{payload = Payload}} ->
-                ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]),
-                true;
-            Msg ->
-                ct:pal("Msg: ~p", [Msg]),
-                false
-        after 100 ->
-            false
-        end
-    ),
-    %% and also the rule should be matched, with matched + 1:
-    {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
-    #{
-        <<"id">> := RuleId,
-        <<"metrics">> := #{
-            <<"matched">> := 1,
-            <<"passed">> := 1,
-            <<"failed">> := 0,
-            <<"failed.exception">> := 0,
-            <<"failed.no_result">> := 0,
-            <<"matched.rate">> := _,
-            <<"matched.rate.max">> := _,
-            <<"matched.rate.last5m">> := _,
-            <<"actions.total">> := 1,
-            <<"actions.success">> := 1,
-            <<"actions.failed">> := 0,
-            <<"actions.failed.out_of_service">> := 0,
-            <<"actions.failed.unknown">> := 0
-        }
-    } = jsx:decode(Rule1),
-    %% we also check if the actions of the rule is triggered
-    ?assertMatch(
-        #{
-            inspect := #{
-                event := <<"$bridges/mqtt", _/binary>>,
-                id := MsgId,
-                payload := Payload,
-                topic := RemoteTopic,
-                qos := 0,
-                dup := false,
-                retain := false,
-                pub_props := #{},
-                timestamp := _
-            }
-        } when is_binary(MsgId),
-        persistent_term:get(?MODULE)
-    ),
-
-    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
-    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
-
-t_egress_mqtt_bridge_with_rules(_) ->
-    {ok, 201, _} = request(
-        post,
-        uri(["connectors"]),
-        ?MQTT_CONNECTOR(<<"user1">>)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?CONNECTR_NAME
-        }
-    ),
-    ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
-    {ok, 201, Bridge} = request(
-        post,
-        uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
-            <<"type">> => ?CONNECTR_TYPE,
-            <<"name">> => ?BRIDGE_NAME_EGRESS
-        }
-    ),
-    #{<<"type">> := ?CONNECTR_TYPE, <<"name">> := ?BRIDGE_NAME_EGRESS} = jsx:decode(Bridge),
-    BridgeIDEgress = emqx_bridge_resource:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
-
-    {ok, 201, Rule} = request(
-        post,
-        uri(["rules"]),
-        #{
-            <<"name">> => <<"A_rule_send_messages_to_a_sink_mqtt_bridge">>,
-            <<"enable">> => true,
-            <<"actions">> => [BridgeIDEgress],
-            <<"sql">> => <<"SELECT * from \"t/1\"">>
-        }
-    ),
-    #{<<"id">> := RuleId} = jsx:decode(Rule),
-
-    %% we now test if the bridge works as expected
-    LocalTopic = <<"local_topic/1">>,
-    RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
-    Payload = <<"hello">>,
-    emqx:subscribe(RemoteTopic),
-    timer:sleep(100),
-    %% PUBLISH a message to the 'local' broker, as we have only one broker,
-    %% the remote broker is also the local one.
-    wait_for_resource_ready(BridgeIDEgress, 5),
-    emqx:publish(emqx_message:make(LocalTopic, Payload)),
-    %% we should receive a message on the "remote" broker, with specified topic
-    ?assert(
-        receive
-            {deliver, RemoteTopic, #message{payload = Payload}} ->
-                ct:pal("remote broker got message: ~p on topic ~p", [Payload, RemoteTopic]),
-                true;
-            Msg ->
-                ct:pal("Msg: ~p", [Msg]),
-                false
-        after 100 ->
-            false
-        end
-    ),
-    emqx:unsubscribe(RemoteTopic),
-
-    %% PUBLISH a message to the rule.
-    Payload2 = <<"hi">>,
-    RuleTopic = <<"t/1">>,
-    RemoteTopic2 = <<"remote_topic/", RuleTopic/binary>>,
-    emqx:subscribe(RemoteTopic2),
-    timer:sleep(100),
-    wait_for_resource_ready(BridgeIDEgress, 5),
-    emqx:publish(emqx_message:make(RuleTopic, Payload2)),
-    {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
-    #{
-        <<"id">> := RuleId,
-        <<"metrics">> := #{
-            <<"matched">> := 1,
-            <<"passed">> := 1,
-            <<"failed">> := 0,
-            <<"failed.exception">> := 0,
-            <<"failed.no_result">> := 0,
-            <<"matched.rate">> := _,
-            <<"matched.rate.max">> := _,
-            <<"matched.rate.last5m">> := _,
-            <<"actions.total">> := 1,
-            <<"actions.success">> := 1,
-            <<"actions.failed">> := 0,
-            <<"actions.failed.out_of_service">> := 0,
-            <<"actions.failed.unknown">> := 0
-        }
-    } = jsx:decode(Rule1),
-    %% we should receive a message on the "remote" broker, with specified topic
-    ?assert(
-        receive
-            {deliver, RemoteTopic2, #message{payload = Payload2}} ->
-                ct:pal("remote broker got message: ~p on topic ~p", [Payload2, RemoteTopic2]),
-                true;
-            Msg ->
-                ct:pal("Msg: ~p", [Msg]),
-                false
-        after 100 ->
-            false
-        end
-    ),
-
-    %% verify the metrics of the bridge
-    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
-    ?assertMatch(
-        #{
-            <<"metrics">> := ?metrics(2, 2, 0, _, _, _),
-            <<"node_metrics">> :=
-                [#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}]
-        },
-        jsx:decode(BridgeStr)
-    ),
-
-    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
-    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
-
-request(Method, Url, Body) ->
-    request(<<"connector_admin">>, Method, Url, Body).
-
-wait_for_resource_ready(InstId, 0) ->
-    ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]),
-    ct:fail(wait_resource_timeout);
-wait_for_resource_ready(InstId, Retry) ->
-    case emqx_bridge:lookup(InstId) of
-        {ok, #{resource_data := #{status := connected}}} ->
-            ok;
-        _ ->
-            timer:sleep(100),
-            wait_for_resource_ready(InstId, Retry - 1)
-    end.

From 45352206a364363770c7c168322d1b6091a291ea Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 22 Aug 2022 18:24:26 +0800
Subject: [PATCH 087/232] fix(auth): remove emqx_connector from testcases of
 authz/authn

---
 apps/emqx_authn/src/emqx_authn.app.src        |  2 +-
 .../test/emqx_authn_mongo_SUITE.erl           |  4 +-
 .../test/emqx_authn_mongo_tls_SUITE.erl       |  4 +-
 .../test/emqx_authn_mysql_SUITE.erl           |  4 +-
 .../test/emqx_authn_mysql_tls_SUITE.erl       |  4 +-
 .../test/emqx_authn_pgsql_SUITE.erl           |  4 +-
 .../test/emqx_authn_pgsql_tls_SUITE.erl       |  4 +-
 .../test/emqx_authn_redis_SUITE.erl           |  4 +-
 .../test/emqx_authn_redis_tls_SUITE.erl       |  4 +-
 apps/emqx_authz/test/emqx_authz_SUITE.erl     |  4 +-
 .../test/emqx_authz_api_cache_SUITE.erl       |  2 +-
 .../test/emqx_authz_api_settings_SUITE.erl    |  2 +-
 .../test/emqx_authz_api_sources_SUITE.erl     |  6 +--
 .../emqx_authz/test/emqx_authz_file_SUITE.erl |  2 +-
 .../emqx_authz/test/emqx_authz_http_SUITE.erl |  6 +--
 .../test/emqx_authz_mongodb_SUITE.erl         |  6 +--
 .../test/emqx_authz_mysql_SUITE.erl           |  6 +--
 .../test/emqx_authz_postgresql_SUITE.erl      |  6 +--
 .../test/emqx_authz_redis_SUITE.erl           |  6 +--
 .../i18n/emqx_bridge_mqtt_schema.conf         | 10 ++++
 .../src/schema/emqx_bridge_mqtt_schema.erl    |  2 +
 .../i18n/emqx_connector_mqtt_schema.conf      |  4 +-
 .../src/mqtt/emqx_connector_mqtt_schema.erl   | 14 ++++--
 .../test/emqx_connector_mongo_SUITE.erl       |  6 ++-
 .../test/emqx_connector_mqtt_worker_tests.erl | 48 -------------------
 .../test/emqx_connector_mysql_SUITE.erl       |  6 ++-
 .../test/emqx_connector_pgsql_SUITE.erl       |  6 ++-
 .../test/emqx_connector_redis_SUITE.erl       |  6 ++-
 apps/emqx_machine/test/emqx_machine_SUITE.erl |  1 -
 29 files changed, 80 insertions(+), 103 deletions(-)

diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index ef67b9a14..65a6c74ff 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -4,7 +4,7 @@
     {vsn, "0.1.4"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
-    {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]},
+    {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},
     {mod, {emqx_authn_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},
diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
index ddde18c49..ee5091283 100644
--- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
@@ -50,7 +50,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?MONGO_HOST, ?MONGO_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             Config;
         false ->
             {skip, no_mongo}
@@ -61,7 +61,7 @@ end_per_suite(_Config) ->
         [authentication],
         ?GLOBAL
     ),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl
index ebece6c3e..1e9981d11 100644
--- a/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl
@@ -46,7 +46,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?MONGO_HOST, ?MONGO_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             Config;
         false ->
             {skip, no_mongo}
@@ -57,7 +57,7 @@ end_per_suite(_Config) ->
         [authentication],
         ?GLOBAL
     ),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
index 175aa7f1d..b9ce9fff2 100644
--- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
@@ -58,7 +58,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             {ok, _} = emqx_resource:create_local(
                 ?MYSQL_RESOURCE,
                 ?RESOURCE_GROUP,
@@ -77,7 +77,7 @@ end_per_suite(_Config) ->
         ?GLOBAL
     ),
     ok = emqx_resource:remove_local(?MYSQL_RESOURCE),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl
index 7d642c230..653be8daa 100644
--- a/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mysql_tls_SUITE.erl
@@ -49,7 +49,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             Config;
         false ->
             {skip, no_mysql_tls}
@@ -60,7 +60,7 @@ end_per_suite(_Config) ->
         [authentication],
         ?GLOBAL
     ),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
index 02095c07d..19b949dd9 100644
--- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
@@ -59,7 +59,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             {ok, _} = emqx_resource:create_local(
                 ?PGSQL_RESOURCE,
                 ?RESOURCE_GROUP,
@@ -78,7 +78,7 @@ end_per_suite(_Config) ->
         ?GLOBAL
     ),
     ok = emqx_resource:remove_local(?PGSQL_RESOURCE),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl
index e2b44b93d..59d37ba96 100644
--- a/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_pgsql_tls_SUITE.erl
@@ -49,7 +49,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             Config;
         false ->
             {skip, no_pgsql_tls}
@@ -60,7 +60,7 @@ end_per_suite(_Config) ->
         [authentication],
         ?GLOBAL
     ),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
index 889404c5e..75a57b1d8 100644
--- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
@@ -58,7 +58,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?REDIS_HOST, ?REDIS_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             {ok, _} = emqx_resource:create_local(
                 ?REDIS_RESOURCE,
                 ?RESOURCE_GROUP,
@@ -77,7 +77,7 @@ end_per_suite(_Config) ->
         ?GLOBAL
     ),
     ok = emqx_resource:remove_local(?REDIS_RESOURCE),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl
index 781d84d98..601b58c3c 100644
--- a/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_redis_tls_SUITE.erl
@@ -49,7 +49,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?REDIS_HOST, ?REDIS_TLS_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_authn]),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             Config;
         false ->
             {skip, no_redis}
@@ -60,7 +60,7 @@ end_per_suite(_Config) ->
         [authentication],
         ?GLOBAL
     ),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl
index 3ee1a94e8..8c18da51f 100644
--- a/apps/emqx_authz/test/emqx_authz_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl
@@ -42,7 +42,7 @@ init_per_suite(Config) ->
     ),
 
     ok = emqx_common_test_helpers:start_apps(
-        [emqx_connector, emqx_conf, emqx_authz],
+        [emqx_conf, emqx_authz],
         fun set_special_configs/1
     ),
     Config.
@@ -57,7 +57,7 @@ end_per_suite(_Config) ->
         }
     ),
     ok = stop_apps([emqx_resource]),
-    emqx_common_test_helpers:stop_apps([emqx_connector, emqx_authz, emqx_conf]),
+    emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]),
     meck:unload(emqx_resource),
     ok.
 
diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
index 0c49cc03a..5844c3b97 100644
--- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
@@ -47,7 +47,7 @@ end_per_suite(_Config) ->
             <<"sources">> => []
         }
     ),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf, emqx_management]),
     ok.
 
diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl
index b53b7aa1b..186f04740 100644
--- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl
@@ -45,7 +45,7 @@ end_per_suite(_Config) ->
             <<"sources">> => []
         }
     ),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
     ok.
 
diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
index 3357798eb..04713090e 100644
--- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
@@ -103,7 +103,7 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
     meck:expect(emqx_resource, create_local, fun(_, _, _, _) -> {ok, meck_data} end),
     meck:expect(emqx_resource, health_check, fun(St) -> {ok, St} end),
@@ -120,7 +120,7 @@ init_per_suite(Config) ->
         [emqx_conf, emqx_authz, emqx_dashboard],
         fun set_special_configs/1
     ),
-    ok = start_apps([emqx_resource, emqx_connector]),
+    ok = start_apps([emqx_resource]),
     Config.
 
 end_per_suite(_Config) ->
@@ -134,7 +134,7 @@ end_per_suite(_Config) ->
     ),
     %% resource and connector should be stop first,
     %% or authz_[mysql|pgsql|redis..]_SUITE would be failed
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]),
     meck:unload(emqx_resource),
     ok.
diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
index 059a350e2..0f04d6b6d 100644
--- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
@@ -55,7 +55,7 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
index 4b5ad7cbf..787012fb7 100644
--- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
@@ -39,17 +39,17 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    ok = stop_apps([emqx_resource, emqx_connector, cowboy]),
+    ok = stop_apps([emqx_resource, cowboy]),
     ok = emqx_common_test_helpers:start_apps(
         [emqx_conf, emqx_authz],
         fun set_special_configs/1
     ),
-    ok = start_apps([emqx_resource, emqx_connector, cowboy]),
+    ok = start_apps([emqx_resource, cowboy]),
     Config.
 
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
-    ok = stop_apps([emqx_resource, emqx_connector, cowboy]),
+    ok = stop_apps([emqx_resource, cowboy]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 set_special_configs(emqx_authz) ->
diff --git a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
index 5e5a6ca1e..7e415f5d3 100644
--- a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
@@ -34,14 +34,14 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     case emqx_common_test_helpers:is_tcp_server_available(?MONGO_HOST, ?MONGO_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps(
                 [emqx_conf, emqx_authz],
                 fun set_special_configs/1
             ),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             Config;
         false ->
             {skip, no_mongo}
@@ -49,7 +49,7 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 set_special_configs(emqx_authz) ->
diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
index ce8d03984..15bd61d06 100644
--- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
@@ -33,14 +33,14 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps(
                 [emqx_conf, emqx_authz],
                 fun set_special_configs/1
             ),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             {ok, _} = emqx_resource:create_local(
                 ?MYSQL_RESOURCE,
                 ?RESOURCE_GROUP,
@@ -56,7 +56,7 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_resource:remove_local(?MYSQL_RESOURCE),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
index d4aaf7077..0059faeb1 100644
--- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
@@ -33,14 +33,14 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     case emqx_common_test_helpers:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps(
                 [emqx_conf, emqx_authz],
                 fun set_special_configs/1
             ),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             {ok, _} = emqx_resource:create_local(
                 ?PGSQL_RESOURCE,
                 ?RESOURCE_GROUP,
@@ -56,7 +56,7 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_resource:remove_local(?PGSQL_RESOURCE),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
index 1b21936b4..ebf2b4d06 100644
--- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
@@ -34,14 +34,14 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     case emqx_common_test_helpers:is_tcp_server_available(?REDIS_HOST, ?REDIS_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps(
                 [emqx_conf, emqx_authz],
                 fun set_special_configs/1
             ),
-            ok = start_apps([emqx_resource, emqx_connector]),
+            ok = start_apps([emqx_resource]),
             {ok, _} = emqx_resource:create_local(
                 ?REDIS_RESOURCE,
                 ?RESOURCE_GROUP,
@@ -57,7 +57,7 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_resource:remove_local(?REDIS_RESOURCE),
-    ok = stop_apps([emqx_resource, emqx_connector]),
+    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
diff --git a/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf
index ab8c97ce7..b935b360c 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf
@@ -1,4 +1,14 @@
 emqx_bridge_mqtt_schema {
+    config {
+        desc {
+                en: """The config for MQTT Bridges."""
+                zh: """MQTT Bridge 的配置。"""
+        }
+        label: {
+                en: "Config"
+                zh: "配置"
+        }
+    }
     desc_type {
                    desc {
                          en: """The bridge type."""
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
index 1bf5565e9..973fc8192 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
@@ -25,6 +25,8 @@ fields("put") ->
 fields("get") ->
     emqx_bridge_schema:metrics_status_fields() ++ fields("config").
 
+desc("config") ->
+    ?DESC("config");
 desc(_) ->
     undefined.
 
diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
index 6dfd0ef27..f92446fe4 100644
--- a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
+++ b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
@@ -71,8 +71,8 @@ is configured, then both the data got from the rule and the MQTT messages that m
 
     egress_local {
         desc {
-                en: """The configs about receiving messages from ben."""
-                zh: """收取本地 Broker 消息相关的配置。"""
+                en: """The configs about receiving messages from local broker."""
+                zh: """如何从本地 Broker 接收消息相关的配置。"""
         }
         label: {
                 en: "Local Configs"
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index d6c6b1fb7..51144748b 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -40,15 +40,21 @@ roots() ->
 fields("config") ->
     fields("server_configs") ++
         [
-            {ingress,
+            {"ingress",
                 mk(
                     hoconsc:union([none, ref(?MODULE, "ingress")]),
-                    #{default => undefined}
+                    #{
+                        default => undefined,
+                        desc => ?DESC("ingress_desc")
+                    }
                 )},
-            {egress,
+            {"egress",
                 mk(
                     hoconsc:union([none, ref(?MODULE, "egress")]),
-                    #{default => undefined}
+                    #{
+                        default => undefined,
+                        desc => ?DESC("egress_desc")
+                    }
                 )}
         ];
 fields("server_configs") ->
diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
index e918be84a..5473463ec 100644
--- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
@@ -36,7 +36,8 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?MONGO_HOST, ?MONGO_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_connector]),
+            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+            {ok, _} = application:ensure_all_started(emqx_connector),
             Config;
         false ->
             {skip, no_mongo}
@@ -44,7 +45,8 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_resource, emqx_connector]).
+    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
+    _ = application:stop(emqx_connector).
 
 init_per_testcase(_, Config) ->
     Config.
diff --git a/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl
index aff1a92a6..3f0374d26 100644
--- a/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl
+++ b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl
@@ -45,22 +45,6 @@ send(SendFun, Batch) when is_function(SendFun, 2) ->
 
 stop(_Pid) -> ok.
 
-%% bridge worker should retry connecting remote node indefinitely
-% reconnect_test() ->
-%     emqx_metrics:start_link(),
-%     emqx_connector_mqtt_worker:register_metrics(),
-%     Ref = make_ref(),
-%     Config = make_config(Ref, self(), {error, test}),
-%     {ok, Pid} = emqx_connector_mqtt_worker:start_link(?BRIDGE_NAME, Config),
-%     %% assert name registered
-%     ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
-%     ?WAIT({connection_start_attempt, Ref}, 1000),
-%     %% expect same message again
-%     ?WAIT({connection_start_attempt, Ref}, 1000),
-%     ok = emqx_connector_mqtt_worker:stop(?BRIDGE_REG_NAME),
-%     emqx_metrics:stop(),
-%     ok.
-
 %% connect first, disconnect, then connect again
 disturbance_test() ->
     meck:new(emqx_connector_mqtt_mod, [passthrough, no_history]),
@@ -69,7 +53,6 @@ disturbance_test() ->
     meck:expect(emqx_connector_mqtt_mod, stop, 1, fun(Pid) -> stop(Pid) end),
     try
         emqx_metrics:start_link(),
-        emqx_connector_mqtt_worker:register_metrics(),
         Ref = make_ref(),
         TestPid = self(),
         Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}),
@@ -84,36 +67,6 @@ disturbance_test() ->
         meck:unload(emqx_connector_mqtt_mod)
     end.
 
-% % %% buffer should continue taking in messages when disconnected
-% buffer_when_disconnected_test_() ->
-%     {timeout, 10000, fun test_buffer_when_disconnected/0}.
-
-% test_buffer_when_disconnected() ->
-%     Ref = make_ref(),
-%     Nums = lists:seq(1, 100),
-%     Sender = spawn_link(fun() -> receive {bridge, Pid} -> sender_loop(Pid, Nums, _Interval = 5) end end),
-%     SenderMref = monitor(process, Sender),
-%     Receiver = spawn_link(fun() -> receive {bridge, Pid} -> receiver_loop(Pid, Nums, _Interval = 1) end end),
-%     ReceiverMref = monitor(process, Receiver),
-%     SendFun = fun(Batch) ->
-%                       BatchRef = make_ref(),
-%                       Receiver ! {batch, BatchRef, Batch},
-%                       {ok, BatchRef}
-%               end,
-%     Config0 = make_config(Ref, false, {ok, #{client_pid => undefined}}),
-%     Config = Config0#{reconnect_delay_ms => 100},
-%     emqx_metrics:start_link(),
-%     emqx_connector_mqtt_worker:register_metrics(),
-%     {ok, Pid} = emqx_connector_mqtt_worker:start_link(?BRIDGE_NAME, Config),
-%     Sender ! {bridge, Pid},
-%     Receiver ! {bridge, Pid},
-%     ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)),
-%     Pid ! {disconnected, Ref, test},
-%     ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000),
-%     ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000),
-%     ok = emqx_connector_mqtt_worker:stop(?BRIDGE_REG_NAME),
-%     emqx_metrics:stop().
-
 manual_start_stop_test() ->
     meck:new(emqx_connector_mqtt_mod, [passthrough, no_history]),
     meck:expect(emqx_connector_mqtt_mod, start, 1, fun(Conf) -> start(Conf) end),
@@ -121,7 +74,6 @@ manual_start_stop_test() ->
     meck:expect(emqx_connector_mqtt_mod, stop, 1, fun(Pid) -> stop(Pid) end),
     try
         emqx_metrics:start_link(),
-        emqx_connector_mqtt_worker:register_metrics(),
         Ref = make_ref(),
         TestPid = self(),
         BridgeName = manual_start_stop,
diff --git a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl
index 3fd7191b9..3a41cc0b1 100644
--- a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl
@@ -36,7 +36,8 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_connector]),
+            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+            {ok, _} = application:ensure_all_started(emqx_connector),
             Config;
         false ->
             {skip, no_mysql}
@@ -44,7 +45,8 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_resource, emqx_connector]).
+    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
+    _ = application:stop(emqx_connector).
 
 init_per_testcase(_, Config) ->
     Config.
diff --git a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl
index 9442a1810..10293a241 100644
--- a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl
@@ -36,7 +36,8 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_connector]),
+            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+            {ok, _} = application:ensure_all_started(emqx_connector),
             Config;
         false ->
             {skip, no_pgsql}
@@ -44,7 +45,8 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_resource, emqx_connector]).
+    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
+    _ = application:stop(emqx_connector).
 
 init_per_testcase(_, Config) ->
     Config.
diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl
index a60702036..d9199d2d6 100644
--- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl
@@ -46,14 +46,16 @@ init_per_suite(Config) ->
     of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_connector]),
+            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+            {ok, _} = application:ensure_all_started(emqx_connector),
             Config;
         false ->
             {skip, no_redis}
     end.
 
 end_per_suite(_Config) ->
-    ok = emqx_common_test_helpers:stop_apps([emqx_resource, emqx_connector]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_resource]),
+    _ = application:stop(emqx_connector).
 
 init_per_testcase(_, Config) ->
     Config.
diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl
index f865b0f26..3e0274337 100644
--- a/apps/emqx_machine/test/emqx_machine_SUITE.erl
+++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl
@@ -47,7 +47,6 @@ init_per_suite(Config) ->
         emqx_prometheus,
         emqx_modules,
         emqx_dashboard,
-        emqx_connector,
         emqx_gateway,
         emqx_statsd,
         emqx_resource,

From 1ff53ee8a9ae6e1e97e951b66cb07db20abff972 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 23 Aug 2022 08:57:12 +0800
Subject: [PATCH 088/232] fix(authz): don't stop emqx_resource app in test
 cases

---
 apps/emqx_authz/src/emqx_authz.app.src              | 1 +
 apps/emqx_authz/test/emqx_authz_SUITE.erl           | 1 -
 apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl | 1 -
 apps/emqx_authz/test/emqx_authz_file_SUITE.erl      | 1 -
 apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src  | 2 +-
 5 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index e40b5e64c..940d73d0d 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -8,6 +8,7 @@
         kernel,
         stdlib,
         crypto,
+        emqx_resource,
         emqx_connector
     ]},
     {env, []},
diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl
index 8c18da51f..1b94441d0 100644
--- a/apps/emqx_authz/test/emqx_authz_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl
@@ -56,7 +56,6 @@ end_per_suite(_Config) ->
             <<"sources">> => []
         }
     ),
-    ok = stop_apps([emqx_resource]),
     emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]),
     meck:unload(emqx_resource),
     ok.
diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
index 5844c3b97..7ddab7321 100644
--- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl
@@ -47,7 +47,6 @@ end_per_suite(_Config) ->
             <<"sources">> => []
         }
     ),
-    ok = stop_apps([emqx_resource]),
     emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf, emqx_management]),
     ok.
 
diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
index 0f04d6b6d..164271c6d 100644
--- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
@@ -55,7 +55,6 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
-    ok = stop_apps([emqx_resource]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
index 83af847ab..10df22d97 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_plugin_libs, [
     {description, "EMQX Plugin utility libs"},
-    {vsn, "4.3.2"},
+    {vsn, "4.3.3"},
     {modules, []},
     {applications, [kernel, stdlib]},
     {env, []}

From 86577365e41c6867a1151d9d04ac95024337b456 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 22 Aug 2022 19:33:27 +0800
Subject: [PATCH 089/232] fix: use gen_statem:cast/3 for async query

---
 apps/emqx_resource/include/emqx_resource.hrl  |  4 +-
 apps/emqx_resource/src/emqx_resource.erl      | 15 +++-
 .../src/emqx_resource_manager.erl             | 10 ++-
 .../src/emqx_resource_worker.erl              | 81 +++++++++++--------
 .../test/emqx_resource_SUITE.erl              |  5 +-
 5 files changed, 76 insertions(+), 39 deletions(-)

diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 3f2cac46b..fa3f0a0f6 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -23,6 +23,7 @@
 -type resource_state() :: term().
 -type resource_status() :: connected | disconnected | connecting | stopped.
 -type callback_mode() :: always_sync | async_if_possible.
+-type query_mode() :: async | sync | dynamic.
 -type result() :: term().
 -type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined.
 -type query_opts() :: #{
@@ -34,6 +35,7 @@
     id := resource_id(),
     mod := module(),
     callback_mode := callback_mode(),
+    query_mode := query_mode(),
     config := resource_config(),
     state := resource_state(),
     status := resource_status(),
@@ -67,7 +69,7 @@
     batch_time => pos_integer(),
     enable_queue => boolean(),
     queue_max_bytes => pos_integer(),
-    query_mode => async | sync | dynamic,
+    query_mode => query_mode(),
     resume_interval => pos_integer(),
     async_inflight_window => pos_integer()
 }.
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 0295292dd..7f1e689ee 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -18,6 +18,7 @@
 
 -include("emqx_resource.hrl").
 -include("emqx_resource_utils.hrl").
+-include("emqx_resource_errors.hrl").
 
 %% APIs for resource types
 
@@ -254,7 +255,19 @@ query(ResId, Request) ->
 -spec query(resource_id(), Request :: term(), emqx_resource_worker:query_opts()) ->
     Result :: term().
 query(ResId, Request, Opts) ->
-    emqx_resource_worker:query(ResId, Request, Opts).
+    case emqx_resource_manager:ets_lookup(ResId) of
+        {ok, _Group, #{query_mode := QM, status := connected}} ->
+            case QM of
+                sync -> emqx_resource_worker:sync_query(ResId, Request, Opts);
+                async -> emqx_resource_worker:async_query(ResId, Request, Opts)
+            end;
+        {ok, _Group, #{status := stopped}} ->
+            ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
+        {ok, _Group, #{status := S}} when S == connecting; S == disconnected ->
+            ?RESOURCE_ERROR(not_connected, "resource not connected");
+        {error, not_found} ->
+            ?RESOURCE_ERROR(not_found, "resource not found")
+    end.
 
 -spec simple_sync_query(resource_id(), Request :: term()) -> Result :: term().
 simple_sync_query(ResId, Request) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 2f6964380..82db194dc 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -53,7 +53,9 @@
 -export([init/1, callback_mode/0, handle_event/4, terminate/3]).
 
 % State record
--record(data, {id, manager_id, group, mod, callback_mode, config, opts, status, state, error}).
+-record(data, {
+    id, manager_id, group, mod, callback_mode, query_mode, config, opts, status, state, error
+}).
 -type data() :: #data{}.
 
 -define(ETS_TABLE, ?MODULE).
@@ -264,6 +266,11 @@ start_link(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         group = Group,
         mod = ResourceType,
         callback_mode = emqx_resource:get_callback_mode(ResourceType),
+        %% query_mode = dynamic | sync | async
+        %% TODO:
+        %%  dynamic mode is async mode when things are going well, but becomes sync mode
+        %%  if the resource worker is overloaded
+        query_mode = maps:get(query_mode, Opts, sync),
         config = Config,
         opts = Opts,
         status = connecting,
@@ -585,6 +592,7 @@ data_record_to_external_map_with_metrics(Data) ->
         id => Data#data.id,
         mod => Data#data.mod,
         callback_mode => Data#data.callback_mode,
+        query_mode => Data#data.query_mode,
         config => Data#data.config,
         status => Data#data.status,
         state => Data#data.state,
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 26b9706c9..e1b51ff12 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -29,7 +29,8 @@
 
 -export([
     start_link/3,
-    query/3,
+    sync_query/3,
+    async_query/3,
     block/1,
     block/2,
     resume/1
@@ -72,12 +73,17 @@ callback_mode() -> [state_functions, state_enter].
 start_link(Id, Index, Opts) ->
     gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []).
 
--spec query(id(), request(), query_opts()) -> Result :: term().
-query(Id, Request, Opts) ->
+-spec sync_query(id(), request(), query_opts()) -> Result :: term().
+sync_query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
     Timeout = maps:get(timeout, Opts, infinity),
     pick_call(Id, PickKey, {query, Request, Opts}, Timeout).
 
+-spec async_query(id(), request(), query_opts()) -> Result :: term().
+async_query(Id, Request, Opts) ->
+    PickKey = maps:get(pick_key, Opts, self()),
+    pick_cast(Id, PickKey, {query, Request, Opts}).
+
 %% simple query the resource without batching and queuing messages.
 -spec simple_sync_query(id(), request()) -> Result :: term().
 simple_sync_query(Id, Request) ->
@@ -125,11 +131,6 @@ init({Id, Index, Opts}) ->
         id => Id,
         index => Index,
         name => Name,
-        %% query_mode = dynamic | sync | async
-        %% TODO:
-        %%  dynamic mode is async mode when things are going well, but becomes sync mode
-        %%  if the resource worker is overloaded
-        query_mode => maps:get(query_mode, Opts, sync),
         async_inflight_window => maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT),
         enable_batch => maps:get(enable_batch, Opts, false),
         batch_size => BatchSize,
@@ -151,9 +152,11 @@ running(cast, block, St) ->
 running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) ->
     Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]),
     {next_state, block, St#{queue := Q1}};
-running({call, From0}, {query, Request, Opts}, #{query_mode := QM} = St) ->
-    From = maybe_quick_return(QM, From0, maps:get(async_reply_fun, Opts, undefined)),
+running({call, From}, {query, Request, _Opts}, St) ->
     query_or_acc(From, Request, St);
+running(cast, {query, Request, Opts}, St) ->
+    ReplayFun = maps:get(async_reply_fun, Opts, undefined),
+    query_or_acc(ReplayFun, Request, St);
 running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) ->
     flush(St#{tref := undefined});
 running(info, {flush, _Ref}, _St) ->
@@ -173,11 +176,15 @@ blocked(cast, resume, St) ->
     do_resume(St);
 blocked(state_timeout, resume, St) ->
     do_resume(St);
-blocked({call, From0}, {query, Request, Opts}, #{id := Id, queue := Q, query_mode := QM} = St) ->
-    From = maybe_quick_return(QM, From0, maps:get(async_reply_fun, Opts, undefined)),
+blocked({call, From}, {query, Request, _Opts}, #{id := Id, queue := Q} = St) ->
     Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
     _ = reply_caller(Id, ?REPLY(From, Request, Error)),
-    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}}.
+    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}};
+blocked(cast, {query, Request, Opts}, #{id := Id, queue := Q} = St) ->
+    ReplayFun = maps:get(async_reply_fun, Opts, undefined),
+    Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
+    _ = reply_caller(Id, ?REPLY(ReplayFun, Request, Error)),
+    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(ReplayFun, Request))])}}.
 
 terminate(_Reason, #{id := Id, index := Index}) ->
     gproc_pool:disconnect_worker(Id, {Id, Index}).
@@ -194,24 +201,25 @@ estimate_size(QItem) ->
     size(queue_item_marshaller(QItem)).
 
 %%==============================================================================
-maybe_quick_return(sync, From, _ReplyFun) ->
-    From;
-maybe_quick_return(async, From, ReplyFun) ->
-    gen_statem:reply(From, ok),
-    ReplyFun.
-
-pick_call(Id, Key, Query, Timeout) ->
-    try gproc_pool:pick_worker(Id, Key) of
+-define(PICK(ID, KEY, EXPR),
+    try gproc_pool:pick_worker(ID, KEY) of
         Pid when is_pid(Pid) ->
-            gen_statem:call(Pid, Query, {clean_timeout, Timeout});
+            EXPR;
         _ ->
-            ?RESOURCE_ERROR(not_created, "resource not found")
+            ?RESOURCE_ERROR(not_created, "resource not created")
     catch
         error:badarg ->
-            ?RESOURCE_ERROR(not_created, "resource not found");
+            ?RESOURCE_ERROR(not_created, "resource not created");
         exit:{timeout, _} ->
             ?RESOURCE_ERROR(timeout, "call resource timeout")
-    end.
+    end
+).
+
+pick_call(Id, Key, Query, Timeout) ->
+    ?PICK(Id, Key, gen_statem:call(Pid, Query, {clean_timeout, Timeout})).
+
+pick_cast(Id, Key, Query) ->
+    ?PICK(Id, Key, gen_statem:cast(Pid, Query)).
 
 do_resume(#{queue := Q, id := Id, name := Name} = St) ->
     case inflight_get_first(Name) of
@@ -264,12 +272,12 @@ query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left
         true -> flush(St);
         false -> {keep_state, ensure_flush_timer(St)}
     end;
-query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id, query_mode := QM} = St) ->
+query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id} = St) ->
     QueryOpts = #{
         inflight_name => maps:get(name, St),
         inflight_window => maps:get(async_inflight_window, St)
     },
-    case send_query(QM, From, Request, Id, QueryOpts) of
+    case send_query(From, Request, Id, QueryOpts) of
         true ->
             Query = ?QUERY(From, Request),
             {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Query)])}};
@@ -277,8 +285,8 @@ query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id, query
             {keep_state, St}
     end.
 
-send_query(QM, From, Request, Id, QueryOpts) ->
-    Result = call_query(QM, Id, ?QUERY(From, Request), QueryOpts),
+send_query(From, Request, Id, QueryOpts) ->
+    Result = call_query(configured, Id, ?QUERY(From, Request), QueryOpts),
     reply_caller(Id, ?REPLY(From, Request, Result)).
 
 flush(#{acc := []} = St) ->
@@ -288,15 +296,14 @@ flush(
         id := Id,
         acc := Batch,
         batch_size := Size,
-        queue := Q0,
-        query_mode := QM
+        queue := Q0
     } = St
 ) ->
     QueryOpts = #{
         inflight_name => maps:get(name, St),
         inflight_window => maps:get(async_inflight_window, St)
     },
-    Result = call_query(QM, Id, Batch, QueryOpts),
+    Result = call_query(configured, Id, Batch, QueryOpts),
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
     case batch_reply_caller(Id, Result, Batch) of
         true ->
@@ -362,9 +369,15 @@ handle_query_result(Id, Result, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, success),
     BlockWorker.
 
-call_query(QM, Id, Query, QueryOpts) ->
+call_query(QM0, Id, Query, QueryOpts) ->
     case emqx_resource_manager:ets_lookup(Id) of
-        {ok, _Group, #{callback_mode := CM, mod := Mod, state := ResSt, status := connected}} ->
+        {ok, _Group, #{mod := Mod, state := ResSt, status := connected} = Data} ->
+            QM =
+                case QM0 of
+                    configured -> maps:get(query_mode, Data);
+                    _ -> QM0
+                end,
+            CM = maps:get(callback_mode, Data),
             apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt, QueryOpts);
         {ok, _Group, #{status := stopped}} ->
             ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index ddd671b75..06160f6c7 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -133,7 +133,7 @@ t_create_remove_local(_) ->
     {error, _} = emqx_resource:remove_local(?ID),
 
     ?assertMatch(
-        ?RESOURCE_ERROR(not_created),
+        ?RESOURCE_ERROR(not_found),
         emqx_resource:query(?ID, get_state)
     ),
     ?assertNot(is_process_alive(Pid)).
@@ -183,7 +183,7 @@ t_query(_) ->
     {ok, #{pid := _}} = emqx_resource:query(?ID, get_state),
 
     ?assertMatch(
-        ?RESOURCE_ERROR(not_created),
+        ?RESOURCE_ERROR(not_found),
         emqx_resource:query(<<"unknown">>, get_state)
     ),
 
@@ -371,6 +371,7 @@ t_query_counter_async_inflight(_) ->
     ok = emqx_resource:query(?ID, {inc_counter, 1}, #{
         async_reply_fun => {Insert, [Tab0, tmp_query]}
     }),
+    timer:sleep(100),
     ?assertMatch([{_, {error, {resource_error, #{reason := blocked}}}}], ets:take(Tab0, tmp_query)),
 
     %% all response should be received after the resource is resumed.

From 43c964c87ea63365e6acf97ee6966e66281536a5 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 23 Aug 2022 18:22:01 +0800
Subject: [PATCH 090/232] fix(docs): ee bridge api docs generation

Use try catch. Because function was unexported before called.
---
 apps/emqx_bridge/src/emqx_bridge_api.erl | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 5fced2467..36f8cf0f5 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -168,9 +168,10 @@ bridge_info_examples(Method) ->
     ).
 
 ee_bridge_examples(Method) ->
-    case erlang:function_exported(emqx_ee_bridge, examples, 1) of
-        true -> emqx_ee_bridge:examples(Method);
-        false -> #{}
+    try
+        emqx_ee_bridge:examples(Method)
+    catch
+        _:_ -> #{}
     end.
 
 info_example(Type, Method) ->

From ca6533395c6c028c177289a2254e4f9b54164a48 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 24 Aug 2022 09:40:08 +0800
Subject: [PATCH 091/232] feat: influxdb bridge structure fits new style

---
 apps/emqx_bridge/src/emqx_bridge.erl          |   8 +-
 .../i18n/emqx_ee_bridge_influxdb.conf         |  20 ---
 .../src/emqx_ee_bridge_influxdb.erl           |  54 +++---
 .../i18n/emqx_ee_connector_influxdb.conf      |  84 ++++-----
 .../include/emqx_ee_connector.hrl             |   5 +
 .../src/emqx_ee_connector_influxdb.erl        | 168 ++++++------------
 6 files changed, 123 insertions(+), 216 deletions(-)
 create mode 100644 lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index 109b1df86..151275bd9 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -47,7 +47,13 @@
 %% exported for `emqx_telemetry'
 -export([get_basic_usage_info/0]).
 
--define(EGRESS_DIR_BRIDGES(T), T == webhook; T == mysql).
+-define(EGRESS_DIR_BRIDGES(T),
+    T == webhook;
+    T == mysql;
+    T == influxdb_api_v1;
+    T == influxdb_api_v2;
+    T == influxdb_udp
+).
 
 load() ->
     Bridges = emqx:get_config([bridges], #{}),
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
index 9e805132e..f34de1119 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -49,16 +49,6 @@ TLDR: </br>
             en: "Enable Or Disable Bridge"
             zh: "启用/禁用桥接"
         }
-        }
-    config_direction {
-        desc {
-            en: """The direction of this bridge, MUST be 'egress'."""
-            zh: """桥接的方向，必须是 egress。"""
-        }
-        label {
-            en: "Bridge Direction"
-            zh: "桥接方向"
-        }
     }
 
     desc_config {
@@ -94,14 +84,4 @@ TLDR: </br>
         }
     }
 
-    desc_connector {
-        desc {
-                en: """Generic configuration for the connector."""
-                zh: """连接器的通用配置。"""
-        }
-        label: {
-                en: "Connector Generic Configuration"
-                zh: "连接器通用配置。"
-        }
-    }
 }
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index dce315721..740ee20ce 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -84,14 +84,6 @@ namespace() -> "bridge_influxdb".
 
 roots() -> [].
 
-fields(basic) ->
-    [
-        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
-        {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
-        {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
-        {write_syntax, fun write_syntax/1}
-    ] ++
-        emqx_resource_schema:fields("resource_opts");
 fields("post_udp") ->
     method_fileds(post, influxdb_udp);
 fields("post_api_v1") ->
@@ -110,35 +102,37 @@ fields("get_api_v1") ->
     method_fileds(get, influxdb_api_v1);
 fields("get_api_v2") ->
     method_fileds(get, influxdb_api_v2);
-fields(Name) when
-    Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2
+fields(Type) when
+    Type == influxdb_udp orelse Type == influxdb_api_v1 orelse Type == influxdb_api_v2
 ->
-    fields(basic) ++
-        connector_field(Name).
+    influxdb_bridge_common_fields() ++
+        connector_fields(Type).
 
 method_fileds(post, ConnectorType) ->
-    fields(basic) ++ connector_field(ConnectorType) ++ type_name_field(ConnectorType);
+    influxdb_bridge_common_fields() ++
+        connector_fields(ConnectorType) ++
+        type_name_fields(ConnectorType);
 method_fileds(get, ConnectorType) ->
-    fields(basic) ++
-        emqx_bridge_schema:metrics_status_fields() ++
-        connector_field(ConnectorType) ++ type_name_field(ConnectorType);
+    influxdb_bridge_common_fields() ++
+        connector_fields(ConnectorType) ++
+        type_name_fields(ConnectorType) ++
+        emqx_bridge_schema:metrics_status_fields();
 method_fileds(put, ConnectorType) ->
-    fields(basic) ++ connector_field(ConnectorType).
+    influxdb_bridge_common_fields() ++
+        connector_fields(ConnectorType).
 
-connector_field(Type) ->
-    [
-        {connector,
-            mk(
-                hoconsc:union([binary(), ref(emqx_ee_connector_influxdb, Type)]),
-                #{
-                    required => true,
-                    example => list_to_binary(atom_to_list(Type) ++ ":connector"),
-                    desc => ?DESC(<<"desc_connector">>)
-                }
-            )}
-    ].
+influxdb_bridge_common_fields() ->
+    emqx_bridge_schema:common_bridge_fields() ++
+        [
+            {local_topic, mk(binary(), #{required => true, desc => ?DESC("local_topic")})},
+            {write_syntax, fun write_syntax/1}
+        ] ++
+        emqx_resource_schema:fields("resource_opts").
 
-type_name_field(Type) ->
+connector_fields(Type) ->
+    emqx_ee_connector_influxdb:fields(Type).
+
+type_name_fields(Type) ->
     [
         {type, mk(Type, #{required => true, desc => ?DESC("desc_type")})},
         {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
index ff2266de5..81ea39d49 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
@@ -1,43 +1,29 @@
 emqx_ee_connector_influxdb {
-    type {
-        desc {
-                en: """The Connector Type."""
-                zh: """连接器类型。"""
-        }
-        label: {
-                en: """Connector Type"""
-                zh: """连接器类型"""
-            }
-    }
 
-    name {
+    server {
         desc {
-                en: """Connector name, used as a human-readable description of the connector."""
-                zh: """连接器名称，人类可读的连接器描述。"""
+          en: """The IPv4 or IPv6 address or the hostname to connect to.</br>
+A host entry has the following form: `Host[:Port]`.</br>
+The InfluxDB default port 8086 is used if `[:Port]` is not specified.
+"""
+          zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。</br>
+主机名具有以下形式：`Host[:Port]`。</br>
+如果未指定 `[:Port]`，则使用 InfluxDB 默认端口 8086。
+"""
+        }
+        label {
+              en: "Server Host"
+              zh: "服务器地址"
         }
-        label: {
-                en: """Connector Name"""
-                zh: """连接器名称"""
-            }
     }
-    host {
+    precision {
         desc {
-                en: """InfluxDB host."""
-                zh: """InfluxDB 主机地址。"""
+                en: """InfluxDB time precision."""
+                zh: """InfluxDB 时间精度。"""
         }
-        label: {
-                en: """Host"""
-                zh: """主机"""
-            }
-    }
-    port {
-        desc {
-                en: """InfluxDB port."""
-                zh: """InfluxDB 端口。"""
-        }
-        label: {
-                en: """Port"""
-                zh: """端口"""
+        label {
+                en: """Time Precision"""
+                zh: """时间精度"""
             }
     }
     protocol {
@@ -45,17 +31,17 @@ emqx_ee_connector_influxdb {
                 en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2."""
                 zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2。"""
         }
-        label: {
+        label {
                 en: """Protocol"""
                 zh: """协议"""
-            }
+        }
     }
     influxdb_udp {
         desc {
                 en: """InfluxDB's UDP protocol."""
                 zh: """InfluxDB UDP 协议。"""
         }
-        label: {
+        label {
                 en: """UDP Protocol"""
                 zh: """UDP 协议"""
             }
@@ -65,7 +51,7 @@ emqx_ee_connector_influxdb {
                 en: """InfluxDB's protocol. Support InfluxDB v1.8 and before."""
                 zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本。"""
         }
-        label: {
+        label {
                 en: """HTTP API Protocol"""
                 zh: """HTTP API 协议"""
             }
@@ -75,7 +61,7 @@ emqx_ee_connector_influxdb {
                 en: """InfluxDB's protocol. Support InfluxDB v2.0 and after."""
                 zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本。"""
         }
-        label: {
+        label {
                 en: """HTTP API V2 Protocol"""
                 zh: """HTTP API V2 协议"""
             }
@@ -85,7 +71,7 @@ emqx_ee_connector_influxdb {
                 en: """InfluxDB database."""
                 zh: """InfluxDB 数据库。"""
         }
-        label: {
+        label {
                 en: "Database"
                 zh: "数据库"
             }
@@ -95,7 +81,7 @@ emqx_ee_connector_influxdb {
                 en: "InfluxDB username."
                 zh: "InfluxDB 用户名。"
         }
-        label: {
+        label {
                 en: "Username"
                 zh: "用户名"
             }
@@ -105,7 +91,7 @@ emqx_ee_connector_influxdb {
                 en: "InfluxDB password."
                 zh: "InfluxDB 密码。"
         }
-        label: {
+        label {
                 en: "Password"
                 zh: "密码"
             }
@@ -115,7 +101,7 @@ emqx_ee_connector_influxdb {
                 en: "InfluxDB bucket name."
                 zh: "InfluxDB bucket 名称。"
         }
-        label: {
+        label {
                 en: "Bucket"
                 zh: "Bucket"
             }
@@ -125,7 +111,7 @@ emqx_ee_connector_influxdb {
                 en: """Organization name of InfluxDB."""
                 zh: """InfluxDB 组织名称。"""
         }
-        label: {
+        label {
                 en: """Organization"""
                 zh: """组织"""
             }
@@ -135,20 +121,10 @@ emqx_ee_connector_influxdb {
                 en: """InfluxDB token."""
                 zh: """InfluxDB token。"""
         }
-        label: {
+        label {
                 en: """Token"""
                 zh: """Token"""
             }
     }
-    precision {
-        desc {
-                en: """InfluxDB time precision."""
-                zh: """InfluxDB 时间精度。"""
-        }
-        label: {
-                en: """Time Precision"""
-                zh: """时间精度"""
-            }
-    }
 
 }
diff --git a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl
new file mode 100644
index 000000000..73807d13a
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl
@@ -0,0 +1,5 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%-------------------------------------------------------------------
+
+-define(INFLUXDB_DEFAULT_PORT, 8086).
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index d2725c797..b83aec4bd 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -3,6 +3,9 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_connector_influxdb).
 
+-include("emqx_ee_connector.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
+
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
@@ -26,10 +29,15 @@
 -export([
     namespace/0,
     fields/1,
-    desc/1,
-    connector_examples/1
+    desc/1
 ]).
 
+%% influxdb servers don't need parse
+-define(INFLUXDB_HOST_OPTIONS, #{
+    host_type => hostname,
+    default_port => ?INFLUXDB_DEFAULT_PORT
+}).
+
 %% -------------------------------------------------------------------------------------------------
 %% resource callback
 callback_mode() -> async_if_possible.
@@ -103,115 +111,41 @@ on_get_status(_InstId, #{client := Client}) ->
 %% schema
 namespace() -> connector_influxdb.
 
-fields("udp_get") ->
-    Key = influxdb_udp,
-    fields(Key) ++ type_name_field(Key);
-fields("udp_post") ->
-    Key = influxdb_udp,
-    fields(Key) ++ type_name_field(Key);
-fields("udp_put") ->
-    fields(influxdb_udp);
-fields("api_v1_get") ->
-    Key = influxdb_api_v1,
-    fields(Key) ++ type_name_field(Key);
-fields("api_v1_post") ->
-    Key = influxdb_api_v1,
-    fields(Key) ++ type_name_field(Key);
-fields("api_v1_put") ->
-    fields(influxdb_api_v1);
-fields("api_v2_get") ->
-    Key = influxdb_api_v2,
-    fields(Key) ++ type_name_field(Key);
-fields("api_v2_post") ->
-    Key = influxdb_api_v2,
-    fields(Key) ++ type_name_field(Key);
-fields("api_v2_put") ->
-    fields(influxdb_api_v2);
-fields(basic) ->
+fields(common) ->
     [
-        {host,
-            mk(binary(), #{required => true, default => <<"127.0.0.1">>, desc => ?DESC("host")})},
-        {port, mk(pos_integer(), #{required => true, default => 8086, desc => ?DESC("port")})},
+        {server, fun server/1},
         {precision,
             mk(enum([ns, us, ms, s, m, h]), #{
                 required => false, default => ms, desc => ?DESC("precision")
             })}
     ];
 fields(influxdb_udp) ->
-    fields(basic);
+    fields(common);
 fields(influxdb_api_v1) ->
-    [
-        {database, mk(binary(), #{required => true, desc => ?DESC("database")})},
-        {username, mk(binary(), #{desc => ?DESC("username")})},
-        {password, mk(binary(), #{desc => ?DESC("password"), format => <<"password">>})}
-    ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic);
+    fields(common) ++
+        [
+            {database, mk(binary(), #{required => true, desc => ?DESC("database")})},
+            {username, mk(binary(), #{desc => ?DESC("username")})},
+            {password, mk(binary(), #{desc => ?DESC("password"), format => <<"password">>})}
+        ] ++ emqx_connector_schema_lib:ssl_fields();
 fields(influxdb_api_v2) ->
-    [
-        {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})},
-        {org, mk(binary(), #{required => true, desc => ?DESC("org")})},
-        {token, mk(binary(), #{required => true, desc => ?DESC("token")})}
-    ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic).
+    fields(common) ++
+        [
+            {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})},
+            {org, mk(binary(), #{required => true, desc => ?DESC("org")})},
+            {token, mk(binary(), #{required => true, desc => ?DESC("token")})}
+        ] ++ emqx_connector_schema_lib:ssl_fields().
 
-type_name_field(Type) ->
-    [
-        {type, mk(Type, #{required => true, desc => ?DESC("type")})},
-        {name, mk(binary(), #{required => true, desc => ?DESC("name")})}
-    ].
-
-connector_examples(Method) ->
-    [
-        #{
-            <<"influxdb_udp">> => #{
-                summary => <<"InfluxDB UDP Connector">>,
-                value => values(udp, Method)
-            }
-        },
-        #{
-            <<"influxdb_api_v1">> => #{
-                summary => <<"InfluxDB HTTP API V1 Connector">>,
-                value => values(api_v1, Method)
-            }
-        },
-        #{
-            <<"influxdb_api_v2">> => #{
-                summary => <<"InfluxDB HTTP API V2 Connector">>,
-                value => values(api_v2, Method)
-            }
-        }
-    ].
-
-values(Protocol, get) ->
-    values(Protocol, post);
-values(Protocol, post) ->
-    Type = list_to_atom("influxdb_" ++ atom_to_list(Protocol)),
-    maps:merge(values(Protocol, put), #{type => Type, name => <<"connector">>});
-values(udp, put) ->
-    #{
-        host => <<"127.0.0.1">>,
-        port => 8089,
-        precision => ms
-    };
-values(api_v1, put) ->
-    #{
-        host => <<"127.0.0.1">>,
-        port => 8086,
-        precision => ms,
-        database => <<"my_db">>,
-        username => <<"my_user">>,
-        password => <<"my_password">>,
-        ssl => #{enable => false}
-    };
-values(api_v2, put) ->
-    #{
-        host => <<"127.0.0.1">>,
-        port => 8086,
-        precision => ms,
-        bucket => <<"my_bucket">>,
-        org => <<"my_org">>,
-        token => <<"my_token">>,
-        ssl => #{enable => false}
-    }.
+server(type) -> emqx_schema:ip_port();
+server(required) -> true;
+server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
+server(converter) -> fun to_server_raw/1;
+server(default) -> <<"127.0.0.1:8086">>;
+server(desc) -> ?DESC("server");
+server(_) -> undefined.
 
+desc(common) ->
+    ?DESC("common");
 desc(influxdb_udp) ->
     ?DESC("influxdb_udp");
 desc(influxdb_api_v1) ->
@@ -248,9 +182,7 @@ do_start_client(
     InstId,
     ClientConfig,
     Config = #{
-        egress := #{
-            write_syntax := Lines
-        }
+        write_syntax := Lines
     }
 ) ->
     case influxdb:start_client(ClientConfig) of
@@ -297,12 +229,11 @@ do_start_client(
 client_config(
     InstId,
     Config = #{
-        host := Host,
-        port := Port
+        server := {Host, Port}
     }
 ) ->
     [
-        {host, binary_to_list(Host)},
+        {host, str(Host)},
         {port, Port},
         {pool_size, erlang:system_info(schedulers)},
         {pool, binary_to_atom(InstId, utf8)},
@@ -319,9 +250,9 @@ protocol_config(#{
     [
         {protocol, http},
         {version, v1},
-        {username, binary_to_list(Username)},
-        {password, binary_to_list(Password)},
-        {database, binary_to_list(DB)}
+        {username, str(Username)},
+        {password, str(Password)},
+        {database, str(DB)}
     ] ++ ssl_config(SSL);
 %% api v1 config
 protocol_config(#{
@@ -333,8 +264,8 @@ protocol_config(#{
     [
         {protocol, http},
         {version, v2},
-        {bucket, binary_to_list(Bucket)},
-        {org, binary_to_list(Org)},
+        {bucket, str(Bucket)},
+        {org, str(Org)},
         {token, Token}
     ] ++ ssl_config(SSL);
 %% udp config
@@ -555,3 +486,18 @@ log_error_points(InstId, Errs) ->
         end,
         Errs
     ).
+
+%% ===================================================================
+%% typereflt funcs
+
+-spec to_server_raw(string()) ->
+    {string(), pos_integer()}.
+to_server_raw(Server) ->
+    emqx_connector_schema_lib:parse_server(Server, ?INFLUXDB_HOST_OPTIONS).
+
+str(A) when is_atom(A) ->
+    atom_to_list(A);
+str(B) when is_binary(B) ->
+    binary_to_list(B);
+str(S) when is_list(S) ->
+    S.

From 4ac6f12252d2992cd9b4748271002fb70ad067c8 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 24 Aug 2022 10:08:03 +0800
Subject: [PATCH 092/232] refactor: configs and APIs for mysql bridge

---
 .../src/emqx_connector_mysql.erl              |  8 +--
 .../i18n/emqx_ee_bridge_mysql.conf            | 20 -------
 .../src/emqx_ee_bridge_mysql.erl              | 57 ++++++++-----------
 3 files changed, 26 insertions(+), 59 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index bedd09267..fdab716cb 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -309,11 +309,9 @@ parse_prepare_sql(Config) ->
     SQL =
         case maps:get(prepare_statement, Config, undefined) of
             undefined ->
-                case emqx_map_lib:deep_get([egress, sql_template], Config, undefined) of
-                    undefined ->
-                        #{};
-                    Template ->
-                        #{send_message => Template}
+                case maps:get(sql, Config, undefined) of
+                    undefined -> #{};
+                    Template -> #{send_message => Template}
                 end;
             Any ->
                 Any
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
index bb908628c..0c56b1976 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf
@@ -38,16 +38,6 @@ will be forwarded.
             zh: "启用/禁用桥接"
         }
         }
-    config_direction {
-        desc {
-            en: """The direction of this bridge, MUST be 'egress'"""
-            zh: """桥接的方向， 必须是 egress"""
-        }
-        label {
-            en: "Bridge Direction"
-            zh: "桥接方向"
-        }
-    }
 
     desc_config {
         desc {
@@ -81,14 +71,4 @@ will be forwarded.
             zh: "桥接名字"
         }
     }
-    desc_connector {
-        desc {
-                en: """Generic configuration for the connector."""
-                zh: """连接器的通用配置。"""
-        }
-        label: {
-                en: "Connector Generic Configuration"
-                zh: "连接器通用配置。"
-        }
-    }
 }
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index e63052d50..cb7390760 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -43,17 +43,17 @@ values(get) ->
     maps:merge(values(post), ?METRICS_EXAMPLE);
 values(post) ->
     #{
+        enable => true,
         type => mysql,
         name => <<"foo">>,
-        sql_template => ?DEFAULT_SQL,
-        connector => #{
-            server => <<"127.0.0.1:3306">>,
-            database => <<"test">>,
-            pool_size => 8,
-            username => <<"root">>,
-            password => <<"">>,
-            auto_reconnect => true
-        },
+        server => <<"127.0.0.1:3306">>,
+        database => <<"test">>,
+        pool_size => 8,
+        username => <<"root">>,
+        password => <<"">>,
+        auto_reconnect => true,
+        sql => ?DEFAULT_SQL,
+        local_topic => <<"local/topic/#">>,
         resource_opts => #{
             health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
             auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
@@ -62,9 +62,7 @@ values(post) ->
             batch_time => ?DEFAULT_BATCH_TIME,
             enable_queue => false,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
-        },
-        enable => true,
-        direction => egress
+        }
     };
 values(put) ->
     values(post).
@@ -78,19 +76,15 @@ roots() -> [].
 fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
-        {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})},
-        {sql_template,
+        {sql,
             mk(
                 binary(),
                 #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>}
             )},
-        {connector,
+        {local_topic,
             mk(
-                ref(?MODULE, connector),
-                #{
-                    required => true,
-                    desc => ?DESC("desc_connector")
-                }
+                binary(),
+                #{desc => ?DESC("local_topic"), default => undefined}
             )},
         {resource_opts,
             mk(
@@ -101,30 +95,20 @@ fields("config") ->
                     desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
                 }
             )}
-    ];
+    ] ++
+        emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields();
 fields("creation_opts") ->
     Opts = emqx_resource_schema:fields("creation_opts"),
-    lists:filter(
-        fun({Field, _}) ->
-            not lists:member(Field, [
-                start_after_created, start_timeout, query_mode, async_inflight_window
-            ])
-        end,
-        Opts
-    );
+    [O || {Field, _} = O <- Opts, not is_hidden_opts(Field)];
 fields("post") ->
     [type_field(), name_field() | fields("config")];
 fields("put") ->
     fields("config");
 fields("get") ->
-    emqx_bridge_schema:metrics_status_fields() ++ fields("post");
-fields(connector) ->
-    emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields().
+    emqx_bridge_schema:metrics_status_fields() ++ fields("post").
 
 desc("config") ->
     ?DESC("desc_config");
-desc(connector) ->
-    ?DESC("desc_connector");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for MySQL using `", string:to_upper(Method), "` method."];
 desc("creation_opts" = Name) ->
@@ -134,6 +118,11 @@ desc(_) ->
 
 %% -------------------------------------------------------------------------------------------------
 %% internal
+is_hidden_opts(Field) ->
+    lists:member(Field, [
+        query_mode, async_inflight_window
+    ]).
+
 type_field() ->
     {type, mk(enum([mysql]), #{required => true, desc => ?DESC("desc_type")})}.
 

From a481e8e2eeef8aab6d534f3c578edbaed2c327a1 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 24 Aug 2022 14:42:19 +0800
Subject: [PATCH 093/232] fix(influxdb): refine influxdb bridge example

---
 .../src/emqx_ee_bridge_influxdb.erl           | 57 +++++++++++++------
 1 file changed, 40 insertions(+), 17 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 740ee20ce..24b4aebb1 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -53,30 +53,53 @@ conn_bridge_examples(Method) ->
 
 values(Protocol, get) ->
     maps:merge(values(Protocol, post), ?METRICS_EXAMPLE);
-values(Protocol, post) ->
-    case Protocol of
-        "influxdb_api_v2" ->
-            SupportUint = <<"uint_value=${payload.uint_key}u,">>;
-        _ ->
-            SupportUint = <<>>
-    end,
-    #{
+values("influxdb_api_v2", post) ->
+    SupportUint = <<"uint_value=${payload.uint_key}u,">>,
+    TypeOpts = #{
+        bucket => <<"example_bucket">>,
+        org => <<"examlpe_org">>,
+        token => <<"example_token">>,
+        server => <<"127.0.0.1:8086">>
+    },
+    values(common, "influxdb_api_v2", SupportUint, TypeOpts);
+values("influxdb_api_v1", post) ->
+    SupportUint = <<>>,
+    TypeOpts = #{
+        database => <<"example_database">>,
+        username => <<"example_username">>,
+        password => <<"examlpe_password">>,
+        server => <<"127.0.0.1:8086">>
+    },
+    values(common, "influxdb_api_v1", SupportUint, TypeOpts);
+values("influxdb_udp", post) ->
+    SupportUint = <<>>,
+    TypeOpts = #{
+        server => <<"127.0.0.1:8089">>
+    },
+    values(common, "influxdb_udp", SupportUint, TypeOpts);
+values(Protocol, put) ->
+    values(Protocol, post).
+
+values(common, Protocol, SupportUint, TypeOpts) ->
+    CommonConfigs = #{
         type => list_to_atom(Protocol),
         name => <<"demo">>,
-        connector => list_to_binary(Protocol ++ ":connector"),
         enable => true,
-        direction => egress,
         local_topic => <<"local/topic/#">>,
         write_syntax =>
             <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
                 "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary,
                 "bool=${payload.bool}">>,
-        enable_batch => false,
-        batch_size => 5,
-        batch_time => <<"1m">>
-    };
-values(Protocol, put) ->
-    values(Protocol, post).
+        precision => ms,
+        resource_opts => #{
+            enable_batch => false,
+            batch_size => 5,
+            batch_time => <<"1m">>
+        },
+        server => <<"127.0.0.1:8086">>,
+        ssl => #{enable => false}
+    },
+    maps:merge(TypeOpts, CommonConfigs).
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
@@ -124,7 +147,7 @@ method_fileds(put, ConnectorType) ->
 influxdb_bridge_common_fields() ->
     emqx_bridge_schema:common_bridge_fields() ++
         [
-            {local_topic, mk(binary(), #{required => true, desc => ?DESC("local_topic")})},
+            {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})},
             {write_syntax, fun write_syntax/1}
         ] ++
         emqx_resource_schema:fields("resource_opts").

From f75932e49f06a8f4437d54ea774f058f73118977 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 24 Aug 2022 22:28:29 +0800
Subject: [PATCH 094/232] chore: release e5.0.0-beta.2

---
 apps/emqx/include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index c843ad92c..1ebf0675c 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -35,7 +35,7 @@
 -define(EMQX_RELEASE_CE, "5.0.5-beta.1").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.0.0-beta.1").
+-define(EMQX_RELEASE_EE, "5.0.0-beta.2").
 
 %% the HTTP API version
 -define(EMQX_API_VERSION, "5.0").

From d854ceb91a366b37e6c9596f5110390e1e5b6f54 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 25 Aug 2022 09:11:22 +0800
Subject: [PATCH 095/232] fix: exclude 'emqx-enterprise' when publish docker
 images

---
 .github/workflows/build_and_push_docker_images.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index 1b724d9e3..a462a62da 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -252,6 +252,8 @@ jobs:
           - 'docker.io'
           - 'public.ecr.aws'
         exclude:
+          - registry: 'public.ecr.aws'
+            profile: emqx-enterprise
           - arch: arm64
             build_machine: ubuntu-20.04
           - arch: amd64

From ddb5bd23d69321a86bfb129ee65a8027383fc4a1 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 25 Aug 2022 10:57:26 +0800
Subject: [PATCH 096/232] fix: don't update manifest if not an exact tag

---
 .github/workflows/build_and_push_docker_images.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index a462a62da..344b694f3 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -221,7 +221,7 @@ jobs:
 
   docker-push-multi-arch-manifest:
     # note, we only run on amd64
-    if: ${{ needs.prepare.outputs.IS_EXACT_TAG }}
+    if: ${{ needs.prepare.outputs.IS_EXACT_TAG }} == 'true'
     needs:
       - prepare
       - docker
@@ -324,7 +324,7 @@ jobs:
             ${{ steps.pre-meta.outputs.img_labels }}
 
       - name: update manifest for multiarch image
-        if: ${{ needs.prepare.outputs.IS_EXACT_TAG }}
+        if: ${{ needs.prepare.outputs.IS_EXACT_TAG }} == 'true'
         working-directory: source
         run: |
           if [ ${{ matrix.build_elixir }} = 'with_elixir' ]; then

From 9327c0f51bf975ba85736d2c46d52ed0bf1c8119 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 25 Aug 2022 11:38:50 +0800
Subject: [PATCH 097/232] fix(mysql_bridge): export the query_mode option to
 the APIs

---
 apps/emqx_rule_engine/src/emqx_rule_runtime.erl    | 4 ++--
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index 3729d9096..2c2bfad4f 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -509,10 +509,10 @@ inc_action_metrics(ok, RuleId) ->
 inc_action_metrics({ok, _}, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success');
 inc_action_metrics({resource_down, _}, RuleId) ->
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.out_of_service'),
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'),
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown');
 inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) ->
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.out_of_service'),
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'),
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown');
 inc_action_metrics(_, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'),
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index cb7390760..e78d77fdb 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -60,6 +60,7 @@ values(post) ->
             enable_batch => false,
             batch_size => ?DEFAULT_BATCH_SIZE,
             batch_time => ?DEFAULT_BATCH_TIME,
+            query_mode => sync,
             enable_queue => false,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
         }
@@ -120,7 +121,7 @@ desc(_) ->
 %% internal
 is_hidden_opts(Field) ->
     lists:member(Field, [
-        query_mode, async_inflight_window
+        async_inflight_window
     ]).
 
 type_field() ->

From edb2e7574f8ac507665802cc5a204e3caec47a9b Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 25 Aug 2022 10:57:26 +0800
Subject: [PATCH 098/232] fix: don't update manifest if not an exact tag

---
 .github/workflows/build_and_push_docker_images.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index a462a62da..fde49eb8b 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -221,7 +221,7 @@ jobs:
 
   docker-push-multi-arch-manifest:
     # note, we only run on amd64
-    if: ${{ needs.prepare.outputs.IS_EXACT_TAG }}
+    if: needs.prepare.outputs.IS_EXACT_TAG == 'true'
     needs:
       - prepare
       - docker
@@ -324,7 +324,7 @@ jobs:
             ${{ steps.pre-meta.outputs.img_labels }}
 
       - name: update manifest for multiarch image
-        if: ${{ needs.prepare.outputs.IS_EXACT_TAG }}
+        if: needs.prepare.outputs.IS_EXACT_TAG == 'true'
         working-directory: source
         run: |
           if [ ${{ matrix.build_elixir }} = 'with_elixir' ]; then

From a896aa8b272db61340f25eceb56e47fdbd74eaac Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 25 Aug 2022 16:06:18 +0800
Subject: [PATCH 099/232] fix: incorrect replayq dir for the emqx_resource

---
 apps/emqx_resource/src/emqx_resource.erl           | 6 +-----
 apps/emqx_resource/src/emqx_resource_worker.erl    | 3 ++-
 apps/emqx_rule_engine/src/emqx_rule_runtime.erl    | 6 ++----
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl | 1 +
 4 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 7f1e689ee..b07460643 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -256,15 +256,11 @@ query(ResId, Request) ->
     Result :: term().
 query(ResId, Request, Opts) ->
     case emqx_resource_manager:ets_lookup(ResId) of
-        {ok, _Group, #{query_mode := QM, status := connected}} ->
+        {ok, _Group, #{query_mode := QM}} ->
             case QM of
                 sync -> emqx_resource_worker:sync_query(ResId, Request, Opts);
                 async -> emqx_resource_worker:async_query(ResId, Request, Opts)
             end;
-        {ok, _Group, #{status := stopped}} ->
-            ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
-        {ok, _Group, #{status := S}} when S == connecting; S == disconnected ->
-            ?RESOURCE_ERROR(not_connected, "resource not connected");
         {error, not_found} ->
             ?RESOURCE_ERROR(not_found, "resource not found")
     end.
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index e1b51ff12..9c632b662 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -526,7 +526,8 @@ name(Id, Index) ->
     list_to_atom(lists:concat([Mod, ":", Id1, ":", Index1])).
 
 disk_queue_dir(Id, Index) ->
-    filename:join([node(), emqx:data_dir(), Id, "queue:" ++ integer_to_list(Index)]).
+    QDir = binary_to_list(Id) ++ ":" ++ integer_to_list(Index),
+    filename:join([emqx:data_dir(), "resource_worker", node(), QDir]).
 
 ensure_flush_timer(St = #{tref := undefined, batch_time := T}) ->
     Ref = make_ref(),
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index 2c2bfad4f..d7fa65829 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -509,11 +509,9 @@ inc_action_metrics(ok, RuleId) ->
 inc_action_metrics({ok, _}, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success');
 inc_action_metrics({resource_down, _}, RuleId) ->
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'),
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown');
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');
 inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) ->
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'),
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown');
+    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');
 inc_action_metrics(_, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'),
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown').
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index e78d77fdb..6a5d4a3a2 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -55,6 +55,7 @@ values(post) ->
         sql => ?DEFAULT_SQL,
         local_topic => <<"local/topic/#">>,
         resource_opts => #{
+            worker_pool_size => 1,
             health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
             auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
             enable_batch => false,

From 6b0ccfbc43196a04ec8a7551f1656d4ce300827b Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 24 Aug 2022 22:18:00 +0800
Subject: [PATCH 100/232] refactor: rename the error return resource_down ->
 recoverable_error

---
 apps/emqx_resource/include/emqx_resource.hrl  |  2 +-
 .../src/emqx_resource_manager.erl             |  2 +-
 .../src/emqx_resource_worker.erl              | 13 ++++++-----
 .../test/emqx_connector_demo.erl              |  2 +-
 .../test/emqx_resource_SUITE.erl              |  2 +-
 .../src/emqx_rule_runtime.erl                 | 22 ++++++++++++++-----
 .../src/emqx_ee_connector_influxdb.erl        | 17 +++++---------
 7 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index fa3f0a0f6..4d1c45eb4 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -77,7 +77,7 @@
     ok
     | {ok, term()}
     | {error, term()}
-    | {resource_down, term()}.
+    | {recoverable_error, term()}.
 
 -define(WORKER_POOL_SIZE, 16).
 
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 82db194dc..f90d042b0 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -128,7 +128,7 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
     ok = emqx_metrics_worker:create_metrics(
         ?RES_METRICS,
         ResId,
-        [matched, success, failed, exception, resource_down],
+        [matched, success, failed, exception],
         [matched]
     ),
     ok = emqx_resource_worker_sup:start_workers(ResId, Opts),
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 9c632b662..8034bcb9e 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -355,13 +355,14 @@ handle_query_result(_Id, ?RESOURCE_ERROR_M(_, _), BlockWorker) ->
 handle_query_result(Id, {error, _}, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
     BlockWorker;
-handle_query_result(Id, {resource_down, _}, _BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down),
+handle_query_result(Id, {recoverable_error, _}, _BlockWorker) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
     true;
 handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
     true;
-handle_query_result(_Id, {async_return, {resource_down, _}}, _BlockWorker) ->
-    true;
+handle_query_result(Id, {async_return, {error, _}}, BlockWorker) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
+    BlockWorker;
 handle_query_result(_Id, {async_return, ok}, BlockWorker) ->
     BlockWorker;
 handle_query_result(Id, Result, BlockWorker) ->
@@ -390,8 +391,8 @@ call_query(QM0, Id, Query, QueryOpts) ->
 -define(APPLY_RESOURCE(EXPR, REQ),
     try
         %% if the callback module (connector) wants to return an error that
-        %% makes the current resource goes into the `error` state, it should
-        %% return `{resource_down, Reason}`
+        %% makes the current resource goes into the `blocked` state, it should
+        %% return `{recoverable_error, Reason}`
         EXPR
     catch
         ERR:REASON:STACKTRACE ->
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index 6e7bca18a..4999b9410 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -96,7 +96,7 @@ on_query(_InstId, {inc_counter, N}, #{pid := Pid}) ->
     Pid ! {From, {inc, N}},
     receive
         {ReqRef, ok} -> ok;
-        {ReqRef, incorrect_status} -> {resource_down, incorrect_status}
+        {ReqRef, incorrect_status} -> {recoverable_error, incorrect_status}
     after 1000 ->
         {error, timeout}
     end;
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 06160f6c7..0bfa67d07 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -419,7 +419,7 @@ t_query_counter_async_inflight(_) ->
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
     ct:pal("metrics: ~p", [C]),
     ?assertMatch(
-        #{matched := M, success := S, exception := E, failed := F, resource_down := RD} when
+        #{matched := M, success := S, exception := E, failed := F, recoverable_error := RD} when
             M >= Sent andalso M == S + E + F + RD,
         C
     ),
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index d7fa65829..aafce137c 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -506,12 +506,22 @@ nested_put(Alias, Val, Columns0) ->
 -define(IS_RES_DOWN(R), R == stopped; R == not_connected; R == not_found).
 inc_action_metrics(ok, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success');
-inc_action_metrics({ok, _}, RuleId) ->
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success');
-inc_action_metrics({resource_down, _}, RuleId) ->
+inc_action_metrics({recoverable_error, _}, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');
 inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');
-inc_action_metrics(_, RuleId) ->
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'),
-    emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown').
+inc_action_metrics(R, RuleId) ->
+    case is_ok_result(R) of
+        false ->
+            emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'),
+            emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown');
+        true ->
+            emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success')
+    end.
+
+is_ok_result(ok) ->
+    true;
+is_ok_result(R) when is_tuple(R) ->
+    ok = erlang:element(1, R);
+is_ok_result(ok) ->
+    false.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index b83aec4bd..2024ba8ce 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -85,18 +85,13 @@ on_batch_query_async(
     InstId,
     BatchData,
     {ReplayFun, Args},
-    State = #{write_syntax := SyntaxLines, client := Client}
+    #{write_syntax := SyntaxLines, client := Client}
 ) ->
-    case on_get_status(InstId, State) of
-        connected ->
-            case parse_batch_data(InstId, BatchData, SyntaxLines) of
-                {ok, Points} ->
-                    do_async_query(InstId, Client, Points, {ReplayFun, Args});
-                {error, Reason} ->
-                    {error, Reason}
-            end;
-        disconnected ->
-            {resource_down, disconnected}
+    case parse_batch_data(InstId, BatchData, SyntaxLines) of
+        {ok, Points} ->
+            do_async_query(InstId, Client, Points, {ReplayFun, Args});
+        {error, Reason} ->
+            {error, Reason}
     end.
 
 on_get_status(_InstId, #{client := Client}) ->

From 1625b8eaebc5a0ca33bbd5a4c97bf9726ab91af3 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 26 Aug 2022 15:50:09 +0800
Subject: [PATCH 101/232] fix(mysql_bridge): export the query_mode option to
 the APIs

---
 .../src/emqx_connector_mysql.erl              |  7 ++
 .../src/emqx_resource_manager.erl             | 15 +++-
 .../src/emqx_resource_worker.erl              | 75 ++++++++++++-------
 3 files changed, 68 insertions(+), 29 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index fdab716cb..b9c200316 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -414,6 +414,13 @@ on_sql_query(
                 LogMeta#{msg => "mysql_connector_prepare_query_failed", reason => not_prepared}
             ),
             Error;
+        {error, {1053, <<"08S01">>, Reason}} ->
+            %% mysql sql server shutdown in progress
+            ?SLOG(
+                error,
+                LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
+            ),
+            {recoverable_error, Reason};
         {error, Reason} ->
             ?SLOG(
                 error,
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index f90d042b0..db4b294a8 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -128,7 +128,20 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
     ok = emqx_metrics_worker:create_metrics(
         ?RES_METRICS,
         ResId,
-        [matched, success, failed, exception],
+        [
+            matched,
+            sent,
+            dropped,
+            queued,
+            batched,
+            inflight,
+            'sent.success',
+            'sent.failed',
+            'sent.exception',
+            'dropped.inflight',
+            'dropped.queued',
+            'dropped.other'
+        ],
         [matched]
     ),
     ok = emqx_resource_worker_sup:start_workers(ResId, Opts),
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 8034bcb9e..8b8b1467c 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -77,23 +77,27 @@ start_link(Id, Index, Opts) ->
 sync_query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
     Timeout = maps:get(timeout, Opts, infinity),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
     pick_call(Id, PickKey, {query, Request, Opts}, Timeout).
 
 -spec async_query(id(), request(), query_opts()) -> Result :: term().
 async_query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
     pick_cast(Id, PickKey, {query, Request, Opts}).
 
 %% simple query the resource without batching and queuing messages.
 -spec simple_sync_query(id(), request()) -> Result :: term().
 simple_sync_query(Id, Request) ->
     Result = call_query(sync, Id, ?QUERY(self(), Request), #{}),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
     _ = handle_query_result(Id, Result, false),
     Result.
 
 -spec simple_async_query(id(), request(), reply_fun()) -> Result :: term().
 simple_async_query(Id, Request, ReplyFun) ->
     Result = call_query(async, Id, ?QUERY(ReplyFun, Request), #{}),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
     _ = handle_query_result(Id, Result, false),
     Result.
 
@@ -149,8 +153,10 @@ running(cast, resume, _St) ->
     keep_state_and_data;
 running(cast, block, St) ->
     {next_state, block, St};
-running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) ->
-    Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]),
+running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{id := Id, queue := Q} = St) when
+    is_list(Batch)
+->
+    Q1 = maybe_append_queue(Id, Q, [?Q_ITEM(Query) || Query <- Batch]),
     {next_state, block, St#{queue := Q1}};
 running({call, From}, {query, Request, _Opts}, St) ->
     query_or_acc(From, Request, St);
@@ -169,8 +175,10 @@ blocked(enter, _, #{resume_interval := ResumeT} = _St) ->
     {keep_state_and_data, {state_timeout, ResumeT, resume}};
 blocked(cast, block, _St) ->
     keep_state_and_data;
-blocked(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) ->
-    Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]),
+blocked(cast, {block, [?QUERY(_, _) | _] = Batch}, #{id := Id, queue := Q} = St) when
+    is_list(Batch)
+->
+    Q1 = maybe_append_queue(Id, Q, [?Q_ITEM(Query) || Query <- Batch]),
     {keep_state, St#{queue := Q1}};
 blocked(cast, resume, St) ->
     do_resume(St);
@@ -179,12 +187,12 @@ blocked(state_timeout, resume, St) ->
 blocked({call, From}, {query, Request, _Opts}, #{id := Id, queue := Q} = St) ->
     Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
     _ = reply_caller(Id, ?REPLY(From, Request, Error)),
-    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}};
+    {keep_state, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(From, Request))])}};
 blocked(cast, {query, Request, Opts}, #{id := Id, queue := Q} = St) ->
     ReplayFun = maps:get(async_reply_fun, Opts, undefined),
     Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
     _ = reply_caller(Id, ?REPLY(ReplayFun, Request, Error)),
-    {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(ReplayFun, Request))])}}.
+    {keep_state, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(ReplayFun, Request))])}}.
 
 terminate(_Reason, #{id := Id, index := Index}) ->
     gproc_pool:disconnect_worker(Id, {Id, Index}).
@@ -206,10 +214,10 @@ estimate_size(QItem) ->
         Pid when is_pid(Pid) ->
             EXPR;
         _ ->
-            ?RESOURCE_ERROR(not_created, "resource not created")
+            ?RESOURCE_ERROR(worker_not_created, "resource not created")
     catch
         error:badarg ->
-            ?RESOURCE_ERROR(not_created, "resource not created");
+            ?RESOURCE_ERROR(worker_not_created, "resource not created");
         exit:{timeout, _} ->
             ?RESOURCE_ERROR(timeout, "call resource timeout")
     end
@@ -277,18 +285,15 @@ query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id} = St)
         inflight_name => maps:get(name, St),
         inflight_window => maps:get(async_inflight_window, St)
     },
-    case send_query(From, Request, Id, QueryOpts) of
+    Result = call_query(configured, Id, ?QUERY(From, Request), QueryOpts),
+    case reply_caller(Id, ?REPLY(From, Request, Result)) of
         true ->
             Query = ?QUERY(From, Request),
-            {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Query)])}};
+            {next_state, blocked, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(Query)])}};
         false ->
             {keep_state, St}
     end.
 
-send_query(From, Request, Id, QueryOpts) ->
-    Result = call_query(configured, Id, ?QUERY(From, Request), QueryOpts),
-    reply_caller(Id, ?REPLY(From, Request, Result)).
-
 flush(#{acc := []} = St) ->
     {keep_state, St};
 flush(
@@ -307,14 +312,18 @@ flush(
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
     case batch_reply_caller(Id, Result, Batch) of
         true ->
-            Q1 = maybe_append_queue(Q0, [?Q_ITEM(Query) || Query <- Batch]),
+            Q1 = maybe_append_queue(Id, Q0, [?Q_ITEM(Query) || Query <- Batch]),
             {next_state, blocked, St1#{queue := Q1}};
         false ->
             {keep_state, St1}
     end.
 
-maybe_append_queue(undefined, _Items) -> undefined;
-maybe_append_queue(Q, Items) -> replayq:append(Q, Items).
+maybe_append_queue(Id, undefined, _Items) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
+    undefined;
+maybe_append_queue(Id, Q, Items) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
+    replayq:append(Q, Items).
 
 batch_reply_caller(Id, BatchResult, Batch) ->
     lists:foldl(
@@ -344,30 +353,40 @@ reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
     handle_query_result(Id, Result, BlockWorker).
 
 handle_query_result(Id, ?RESOURCE_ERROR_M(exception, _), BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, exception),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.exception'),
     BlockWorker;
 handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _) when
     NotWorking == not_connected; NotWorking == blocked
 ->
     true;
-handle_query_result(_Id, ?RESOURCE_ERROR_M(_, _), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, _), BlockWorker) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_not_found'),
+    BlockWorker;
+handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, _), BlockWorker) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_stopped'),
+    BlockWorker;
+handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
+    ?SLOG(error, #{msg => other_resource_error, reason => Reason}),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.other'),
     BlockWorker;
 handle_query_result(Id, {error, _}, BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
-handle_query_result(Id, {recoverable_error, _}, _BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
+handle_query_result(_Id, {recoverable_error, _}, _BlockWorker) ->
     true;
 handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
     true;
 handle_query_result(Id, {async_return, {error, _}}, BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, failed),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
 handle_query_result(_Id, {async_return, ok}, BlockWorker) ->
     BlockWorker;
 handle_query_result(Id, Result, BlockWorker) ->
     assert_ok_result(Result),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, success),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.success'),
     BlockWorker.
 
 call_query(QM0, Id, Query, QueryOpts) ->
@@ -407,7 +426,7 @@ call_query(QM0, Id, Query, QueryOpts) ->
 
 apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent),
     ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request);
 apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
@@ -419,7 +438,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
                 ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent),
                 ReplyFun = fun ?MODULE:reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = [self(), Id, Name, Ref, Query],
@@ -432,7 +451,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
 apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
     ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent, length(Batch)),
     ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch);
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
@@ -444,7 +463,7 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
                 ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent, length(Batch)),
                 ReplyFun = fun ?MODULE:batch_reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = {ReplyFun, [self(), Id, Name, Ref, Batch]},

From d436d8d0151a84e597d49ea4a9cce5dbdad427f2 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 26 Aug 2022 18:13:14 +0800
Subject: [PATCH 102/232] chore: update dashboard for e5.0

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index a363a467d..51f0b33d5 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v1.0.6
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.1
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

From ca7ad9cc15bb2612be568ce4f79a266933f2cce0 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 23 Aug 2022 18:17:32 +0800
Subject: [PATCH 103/232] chore: refactor mqtt connector

---
 .../src/emqx_connector_mqtt.erl               |  11 +
 .../src/mqtt/emqx_connector_mqtt_mod.erl      |  44 +---
 .../src/mqtt/emqx_connector_mqtt_worker.erl   | 229 +++++-------------
 3 files changed, 82 insertions(+), 202 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index bdf43885a..59eb3f0fb 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -39,6 +39,7 @@
     on_start/2,
     on_stop/2,
     on_query/3,
+    on_query_async/4,
     on_get_status/2
 ]).
 
@@ -190,6 +191,16 @@ on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) ->
     emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
     ok.
 
+on_query_async(
+    _InstId,
+    {send_message, Msg},
+    {ReplayFun, Args},
+    #{name := InstanceId}
+) ->
+    ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
+    emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, {ReplayFun, Args}),
+    ok.
+
 on_get_status(_InstId, #{name := InstanceId, bridge_conf := Conf}) ->
     AutoReconn = maps:get(auto_reconnect, Conf, true),
     case emqx_connector_mqtt_worker:status(InstanceId) of
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
index 372405b59..7571c59b8 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
@@ -21,6 +21,7 @@
 -export([
     start/1,
     send/2,
+    send_async/3,
     stop/1,
     ping/1
 ]).
@@ -32,7 +33,6 @@
 
 %% callbacks for emqtt
 -export([
-    handle_puback/2,
     handle_publish/3,
     handle_disconnected/2
 ]).
@@ -134,44 +134,11 @@ safe_stop(Pid, StopF, Timeout) ->
         exit(Pid, kill)
     end.
 
-send(Conn, Msgs) ->
-    send(Conn, Msgs, []).
+send(#{client_pid := ClientPid}, Msg) ->
+    emqtt:publish(ClientPid, Msg).
 
-send(_Conn, [], []) ->
-    %% all messages in the batch are QoS-0
-    Ref = make_ref(),
-    %% QoS-0 messages do not have packet ID
-    %% the batch ack is simulated with a loop-back message
-    self() ! {batch_ack, Ref},
-    {ok, Ref};
-send(_Conn, [], PktIds) ->
-    %% PktIds is not an empty list if there is any non-QoS-0 message in the batch,
-    %% And the worker should wait for all acks
-    {ok, PktIds};
-send(#{client_pid := ClientPid} = Conn, [Msg | Rest], PktIds) ->
-    case emqtt:publish(ClientPid, Msg) of
-        ok ->
-            send(Conn, Rest, PktIds);
-        {ok, PktId} ->
-            send(Conn, Rest, [PktId | PktIds]);
-        {error, Reason} ->
-            %% NOTE: There is no partial success of a batch and recover from the middle
-            %% only to retry all messages in one batch
-            {error, Reason}
-    end.
-
-handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) when
-    RC =:= ?RC_SUCCESS;
-    RC =:= ?RC_NO_MATCHING_SUBSCRIBERS
-->
-    Parent ! {batch_ack, PktId},
-    ok;
-handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
-    ?SLOG(warning, #{
-        msg => "publish_to_remote_node_falied",
-        packet_id => PktId,
-        reason_code => RC
-    }).
+send_async(#{client_pid := ClientPid}, Msg, Callback) ->
+    emqtt:publish_async(ClientPid, Msg, Callback).
 
 handle_publish(Msg, undefined, _Opts) ->
     ?SLOG(error, #{
@@ -200,7 +167,6 @@ handle_disconnected(Reason, Parent) ->
 
 make_hdlr(Parent, Vars, Opts) ->
     #{
-        puback => {fun ?MODULE:handle_puback/2, [Parent]},
         publish => {fun ?MODULE:handle_publish/3, [Vars, Opts]},
         disconnected => {fun ?MODULE:handle_disconnected/2, [Parent]}
     }.
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index 3f3a4b9ce..5e4fa8f72 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -91,16 +91,14 @@
     ensure_stopped/1,
     status/1,
     ping/1,
-    send_to_remote/2
+    send_to_remote/2,
+    send_to_remote_async/3
 ]).
 
 -export([get_forwards/1]).
 
 -export([get_subscriptions/1]).
 
-%% Internal
--export([msg_marshaller/1]).
-
 -export_type([
     config/0,
     ack_ref/0
@@ -133,12 +131,6 @@
 %% mountpoint: The topic mount point for messages sent to remote node/cluster
 %%      `undefined', `<<>>' or `""' to disable
 %% forwards: Local topics to subscribe.
-%% replayq.batch_bytes_limit: Max number of bytes to collect in a batch for each
-%%      send call towards emqx_bridge_connect
-%% replayq.batch_count_limit: Max number of messages to collect in a batch for
-%%      each send call towards emqx_bridge_connect
-%% replayq.dir: Directory where replayq should persist messages
-%% replayq.seg_bytes: Size in bytes for each replayq segment file
 %%
 %% Find more connection specific configs in the callback modules
 %% of emqx_bridge_connect behaviour.
@@ -173,9 +165,14 @@ ping(Name) ->
     gen_statem:call(name(Name), ping).
 
 send_to_remote(Pid, Msg) when is_pid(Pid) ->
-    gen_statem:cast(Pid, {send_to_remote, Msg});
+    gen_statem:call(Pid, {send_to_remote, Msg});
 send_to_remote(Name, Msg) ->
-    gen_statem:cast(name(Name), {send_to_remote, Msg}).
+    gen_statem:call(name(Name), {send_to_remote, Msg}).
+
+send_to_remote_async(Pid, Msg, Callback) when is_pid(Pid) ->
+    gen_statem:cast(Pid, {send_to_remote_async, Msg, Callback});
+send_to_remote_async(Name, Msg, Callback) ->
+    gen_statem:cast(name(Name), {send_to_remote_async, Msg, Callback}).
 
 %% @doc Return all forwards (local subscriptions).
 -spec get_forwards(id()) -> [topic()].
@@ -194,12 +191,10 @@ init(#{name := Name} = ConnectOpts) ->
         name => Name
     }),
     erlang:process_flag(trap_exit, true),
-    Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})),
     State = init_state(ConnectOpts),
     self() ! idle,
     {ok, idle, State#{
-        connect_opts => pre_process_opts(ConnectOpts),
-        replayq => Queue
+        connect_opts => pre_process_opts(ConnectOpts)
     }}.
 
 init_state(Opts) ->
@@ -212,32 +207,11 @@ init_state(Opts) ->
         start_type => StartType,
         reconnect_interval => ReconnDelayMs,
         mountpoint => format_mountpoint(Mountpoint),
-        inflight => [],
         max_inflight => MaxInflightSize,
         connection => undefined,
         name => Name
     }.
 
-open_replayq(Name, QCfg) ->
-    Dir = maps:get(dir, QCfg, undefined),
-    SegBytes = maps:get(seg_bytes, QCfg, ?DEFAULT_SEG_BYTES),
-    MaxTotalSize = maps:get(max_total_size, QCfg, ?DEFAULT_MAX_TOTAL_SIZE),
-    QueueConfig =
-        case Dir =:= undefined orelse Dir =:= "" of
-            true ->
-                #{mem_only => true};
-            false ->
-                #{
-                    dir => filename:join([Dir, node(), Name]),
-                    seg_bytes => SegBytes,
-                    max_total_size => MaxTotalSize
-                }
-        end,
-    replayq:open(QueueConfig#{
-        sizer => fun emqx_connector_mqtt_msg:estimate_size/1,
-        marshaller => fun ?MODULE:msg_marshaller/1
-    }).
-
 pre_process_opts(#{subscriptions := InConf, forwards := OutConf} = ConnectOpts) ->
     ConnectOpts#{
         subscriptions => pre_process_in_out(in, InConf),
@@ -276,9 +250,8 @@ pre_process_conf(Key, Conf) ->
 code_change(_Vsn, State, Data, _Extra) ->
     {ok, State, Data}.
 
-terminate(_Reason, _StateName, #{replayq := Q} = State) ->
+terminate(_Reason, _StateName, State) ->
     _ = disconnect(State),
-    _ = replayq:close(Q),
     maybe_destroy_session(State).
 
 maybe_destroy_session(#{connect_opts := ConnectOpts = #{clean_start := false}} = State) ->
@@ -322,15 +295,18 @@ connecting(#{reconnect_interval := ReconnectDelayMs} = State) ->
             {keep_state_and_data, {state_timeout, ReconnectDelayMs, reconnect}}
     end.
 
-connected(state_timeout, connected, #{inflight := Inflight} = State) ->
-    case retry_inflight(State#{inflight := []}, Inflight) of
-        {ok, NewState} ->
-            {keep_state, NewState, {next_event, internal, maybe_send}};
-        {error, NewState} ->
-            {keep_state, NewState}
+connected(state_timeout, connected, State) ->
+    %% nothing to do
+    {keep_state, State};
+connected({call, From}, {send_to_remote, Msg}, State) ->
+    case do_send(State, Msg) of
+        {ok, NState} ->
+            {keep_state, NState, [{reply, From, ok}]};
+        {error, Reason} ->
+            {keep_state_and_data, [[reply, From, {error, Reason}]]}
     end;
-connected(internal, maybe_send, State) ->
-    {_, NewState} = pop_and_send(State),
+connected(cast, {send_to_remote_async, Msg, Callback}, State) ->
+    {_, NewState} = do_send_async(State, Msg, Callback),
     {keep_state, NewState};
 connected(
     info,
@@ -345,9 +321,6 @@ connected(
         false ->
             keep_state_and_data
     end;
-connected(info, {batch_ack, Ref}, State) ->
-    NewState = handle_batch_ack(State, Ref),
-    {keep_state, NewState, {next_event, internal, maybe_send}};
 connected(Type, Content, State) ->
     common(connected, Type, Content, State).
 
@@ -368,9 +341,6 @@ common(_StateName, {call, From}, get_subscriptions, #{connection := Connection})
     {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, #{})}]};
 common(_StateName, info, {'EXIT', _, _}, State) ->
     {keep_state, State};
-common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) ->
-    NewQ = replayq:append(Q, [Msg]),
-    {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
 common(StateName, Type, Content, #{name := Name} = State) ->
     ?SLOG(notice, #{
         msg => "bridge_discarded_event",
@@ -384,13 +354,12 @@ common(StateName, Type, Content, #{name := Name} = State) ->
 do_connect(
     #{
         connect_opts := ConnectOpts,
-        inflight := Inflight,
         name := Name
     } = State
 ) ->
     case emqx_connector_mqtt_mod:start(ConnectOpts) of
         {ok, Conn} ->
-            ?tp(info, connected, #{name => Name, inflight => length(Inflight)}),
+            ?tp(info, connected, #{name => Name}),
             {ok, State#{connection => Conn}};
         {error, Reason} ->
             ConnectOpts1 = obfuscate(ConnectOpts),
@@ -402,39 +371,7 @@ do_connect(
             {error, Reason, State}
     end.
 
-%% Retry all inflight (previously sent but not acked) batches.
-retry_inflight(State, []) ->
-    {ok, State};
-retry_inflight(State, [#{q_ack_ref := QAckRef, msg := Msg} | Rest] = OldInf) ->
-    case do_send(State, QAckRef, Msg) of
-        {ok, State1} ->
-            retry_inflight(State1, Rest);
-        {error, #{inflight := NewInf} = State1} ->
-            {error, State1#{inflight := NewInf ++ OldInf}}
-    end.
-
-pop_and_send(#{inflight := Inflight, max_inflight := Max} = State) ->
-    pop_and_send_loop(State, Max - length(Inflight)).
-
-pop_and_send_loop(State, 0) ->
-    ?tp(debug, inflight_full, #{}),
-    {ok, State};
-pop_and_send_loop(#{replayq := Q} = State, N) ->
-    case replayq:is_empty(Q) of
-        true ->
-            ?tp(debug, replayq_drained, #{}),
-            {ok, State};
-        false ->
-            BatchSize = 1,
-            Opts = #{count_limit => BatchSize, bytes_limit => 999999999},
-            {Q1, QAckRef, [Msg]} = replayq:pop(Q, Opts),
-            case do_send(State#{replayq := Q1}, QAckRef, Msg) of
-                {ok, NewState} -> pop_and_send_loop(NewState, N - 1);
-                {error, NewState} -> {error, NewState}
-            end
-    end.
-
-do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) ->
+do_send(#{connect_opts := #{forwards := undefined}}, Msg) ->
     ?SLOG(error, #{
         msg =>
             "cannot_forward_messages_to_remote_broker"
@@ -443,98 +380,68 @@ do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) ->
     });
 do_send(
     #{
-        inflight := Inflight,
         connection := Connection,
         mountpoint := Mountpoint,
         connect_opts := #{forwards := Forwards}
     } = State,
-    QAckRef,
     Msg
 ) ->
     Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards),
-    ExportMsg = fun(Message) ->
-        emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
-    end,
+    ExportMsg = emqx_connector_mqtt_msg:to_remote_msg(Msg, Vars),
     ?SLOG(debug, #{
         msg => "publish_to_remote_broker",
         message => Msg,
         vars => Vars
     }),
-    case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of
-        {ok, Refs} ->
-            {ok, State#{
-                inflight := Inflight ++
-                    [
-                        #{
-                            q_ack_ref => QAckRef,
-                            send_ack_ref => map_set(Refs),
-                            msg => Msg
-                        }
-                    ]
-            }};
+    case emqx_connector_mqtt_mod:send(Connection, ExportMsg) of
+        ok ->
+            {ok, State};
+        {ok, #{reason_code := RC}} when
+            RC =:= ?RC_SUCCESS;
+            RC =:= ?RC_NO_MATCHING_SUBSCRIBERS
+        ->
+            {ok, State};
+        {ok, #{reason_code := RC, reason_code_name := RCN}} ->
+            ?SLOG(warning, #{
+                msg => "publish_to_remote_node_falied",
+                message => Msg,
+                reason_code => RC,
+                reason_code_name => RCN
+            }),
+            {error, RCN};
         {error, Reason} ->
             ?SLOG(info, #{
                 msg => "mqtt_bridge_produce_failed",
                 reason => Reason
             }),
-            {error, State}
+            {error, Reason}
     end.
 
-%% map as set, ack-reference -> 1
-map_set(Ref) when is_reference(Ref) ->
-    %% QoS-0 or RPC call returns a reference
-    map_set([Ref]);
-map_set(List) ->
-    map_set(List, #{}).
-
-map_set([], Set) -> Set;
-map_set([H | T], Set) -> map_set(T, Set#{H => 1}).
-
-handle_batch_ack(#{inflight := Inflight0, replayq := Q} = State, Ref) ->
-    Inflight1 = do_ack(Inflight0, Ref),
-    Inflight = drop_acked_batches(Q, Inflight1),
-    State#{inflight := Inflight}.
-
-do_ack([], Ref) ->
-    ?SLOG(debug, #{
-        msg => "stale_batch_ack_reference",
-        ref => Ref
-    }),
-    [];
-do_ack([#{send_ack_ref := Refs} = First | Rest], Ref) ->
-    case maps:is_key(Ref, Refs) of
-        true ->
-            NewRefs = maps:without([Ref], Refs),
-            [First#{send_ack_ref := NewRefs} | Rest];
-        false ->
-            [First | do_ack(Rest, Ref)]
-    end.
-
-%% Drop the consecutive header of the inflight list having empty send_ack_ref
-drop_acked_batches(_Q, []) ->
-    ?tp(debug, inflight_drained, #{}),
-    [];
-drop_acked_batches(
-    Q,
-    [
-        #{
-            send_ack_ref := Refs,
-            q_ack_ref := QAckRef
-        }
-        | Rest
-    ] = All
+do_send_async(#{connect_opts := #{forwards := undefined}}, Msg, _Callback) ->
+    %% TODO: eval callback with undefined error
+    ?SLOG(error, #{
+        msg =>
+            "cannot_forward_messages_to_remote_broker"
+            "_as_'egress'_is_not_configured",
+        messages => Msg
+    });
+do_send_async(
+    #{
+        connection := Connection,
+        mountpoint := Mountpoint,
+        connect_opts := #{forwards := Forwards}
+    },
+    Msg,
+    Callback
 ) ->
-    case maps:size(Refs) of
-        0 ->
-            %% all messages are acked by bridge target
-            %% now it's safe to ack replayq (delete from disk)
-            ok = replayq:ack(Q, QAckRef),
-            %% continue to check more sent batches
-            drop_acked_batches(Q, Rest);
-        _ ->
-            %% the head (oldest) inflight batch is not acked, keep waiting
-            All
-    end.
+    Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards),
+    ExportMsg = emqx_connector_mqtt_msg:to_remote_msg(Msg, Vars),
+    ?SLOG(debug, #{
+        msg => "publish_to_remote_broker",
+        message => Msg,
+        vars => Vars
+    }),
+    emqx_connector_mqtt_mod:send_async(Connection, ExportMsg, Callback).
 
 disconnect(#{connection := Conn} = State) when Conn =/= undefined ->
     emqx_connector_mqtt_mod:stop(Conn),
@@ -542,10 +449,6 @@ disconnect(#{connection := Conn} = State) when Conn =/= undefined ->
 disconnect(State) ->
     State.
 
-%% Called only when replayq needs to dump it to disk.
-msg_marshaller(Bin) when is_binary(Bin) -> emqx_connector_mqtt_msg:from_binary(Bin);
-msg_marshaller(Msg) -> emqx_connector_mqtt_msg:to_binary(Msg).
-
 format_mountpoint(undefined) ->
     undefined;
 format_mountpoint(Prefix) ->

From 0aa10702db157c03e23edd53b7d741ed3b8dcf9e Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 24 Aug 2022 15:53:21 +0800
Subject: [PATCH 104/232] feat(bridge): support async mode resource options

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      |  8 ++++
 .../src/schema/emqx_bridge_mqtt_schema.erl    | 45 ++++++++++---------
 .../src/emqx_connector_mqtt.erl               |  2 +-
 .../src/mqtt/emqx_connector_mqtt_mod.erl      |  2 +-
 .../src/mqtt/emqx_connector_mqtt_worker.erl   |  4 +-
 5 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 36f8cf0f5..848266b43 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -20,6 +20,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("emqx_resource/include/emqx_resource.hrl").
 
 -import(hoconsc, [mk/2, array/1, enum/1]).
 
@@ -237,6 +238,13 @@ mqtt_main_example() ->
         keepalive => <<"300s">>,
         retry_interval => <<"15s">>,
         max_inflight => 100,
+        resource_opts => #{
+            health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
+            auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
+            query_mode => sync,
+            enable_queue => false,
+            max_queue_bytes => ?DEFAULT_QUEUE_SIZE
+        },
         ssl => #{
             enable => false
         }
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
index 973fc8192..8fb6af65c 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
@@ -12,16 +12,29 @@
 namespace() -> "bridge_mqtt".
 
 roots() -> [].
+
 fields("config") ->
+    %% enable
     emqx_bridge_schema:common_bridge_fields() ++
+        [
+            {resource_opts,
+                mk(
+                    ref(?MODULE, "creation_opts"),
+                    #{
+                        required => false,
+                        default => #{},
+                        desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
+                    }
+                )}
+        ] ++
         emqx_connector_mqtt_schema:fields("config");
+fields("creation_opts") ->
+    Opts = emqx_resource_schema:fields("creation_opts"),
+    [O || {Field, _} = O <- Opts, not is_hidden_opts(Field)];
 fields("post") ->
-    [
-        type_field(),
-        name_field()
-    ] ++ emqx_connector_mqtt_schema:fields("config");
+    [type_field(), name_field() | fields("config")];
 fields("put") ->
-    emqx_connector_mqtt_schema:fields("config");
+    fields("config");
 fields("get") ->
     emqx_bridge_schema:metrics_status_fields() ++ fields("config").
 
@@ -31,22 +44,12 @@ desc(_) ->
     undefined.
 
 %%======================================================================================
+%% internal
+is_hidden_opts(Field) ->
+    lists:member(Field, [enable_batch, batch_size, batch_time]).
+
 type_field() ->
-    {type,
-        mk(
-            mqtt,
-            #{
-                required => true,
-                desc => ?DESC("desc_type")
-            }
-        )}.
+    {type, mk(mqtt, #{required => true, desc => ?DESC("desc_type")})}.
 
 name_field() ->
-    {name,
-        mk(
-            binary(),
-            #{
-                required => true,
-                desc => ?DESC("desc_name")
-            }
-        )}.
+    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 59eb3f0fb..0b9f7c85c 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -141,7 +141,7 @@ on_message_received(Msg, HookPoint, ResId) ->
     emqx:run_hook(HookPoint, [Msg]).
 
 %% ===================================================================
-callback_mode() -> always_sync.
+callback_mode() -> async_if_possible.
 
 on_start(InstId, Conf) ->
     InstanceId = binary_to_atom(InstId, utf8),
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
index 7571c59b8..f1ecbf68c 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
@@ -138,7 +138,7 @@ send(#{client_pid := ClientPid}, Msg) ->
     emqtt:publish(ClientPid, Msg).
 
 send_async(#{client_pid := ClientPid}, Msg, Callback) ->
-    emqtt:publish_async(ClientPid, Msg, Callback).
+    emqtt:publish_async(ClientPid, Msg, infinity, Callback).
 
 handle_publish(Msg, undefined, _Opts) ->
     ?SLOG(error, #{
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index 5e4fa8f72..618361ad3 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -306,8 +306,8 @@ connected({call, From}, {send_to_remote, Msg}, State) ->
             {keep_state_and_data, [[reply, From, {error, Reason}]]}
     end;
 connected(cast, {send_to_remote_async, Msg, Callback}, State) ->
-    {_, NewState} = do_send_async(State, Msg, Callback),
-    {keep_state, NewState};
+    _ = do_send_async(State, Msg, Callback),
+    {keep_state, State};
 connected(
     info,
     {disconnected, Conn, Reason},

From a6eff81163916faf409fcee3c680edd1e7a7ad01 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 24 Aug 2022 15:54:06 +0800
Subject: [PATCH 105/232] chore: update emqtt to 1.7.0-rc.1

---
 apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl | 2 ++
 apps/emqx_connector/rebar.config                        | 3 +--
 mix.exs                                                 | 2 +-
 rebar.config                                            | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
index 8fb6af65c..6d2baaaa8 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl
@@ -40,6 +40,8 @@ fields("get") ->
 
 desc("config") ->
     ?DESC("config");
+desc("creation_opts" = Name) ->
+    emqx_resource_schema:desc(Name);
 desc(_) ->
     undefined.
 
diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config
index 0ac2d0da8..4d0b53e9a 100644
--- a/apps/emqx_connector/rebar.config
+++ b/apps/emqx_connector/rebar.config
@@ -20,8 +20,7 @@
     %% By accident, We have always been using the upstream fork due to
     %% eredis_cluster's dependency getting resolved earlier.
     %% Here we pin 1.5.2 to avoid surprises in the future.
-    {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}},
-    {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.6.0"}}}
+    {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}
 ]}.
 
 {shell, [
diff --git a/mix.exs b/mix.exs
index 3a15b36b4..1a4227900 100644
--- a/mix.exs
+++ b/mix.exs
@@ -59,7 +59,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true},
       {:replayq, "0.3.4", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
-      {:emqtt, github: "emqx/emqtt", tag: "1.6.0", override: true},
+      {:emqtt, github: "emqx/emqtt", tag: "1.7.0-rc.1", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
diff --git a/rebar.config b/rebar.config
index 8370278f1..59f2e9101 100644
--- a/rebar.config
+++ b/rebar.config
@@ -61,7 +61,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
     , {replayq, "0.3.4"}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.6.0"}}}
+    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0-rc.1"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}

From 6fde37791c3df2dbef49c054323d637140a1e506 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 10:14:10 +0800
Subject: [PATCH 106/232] refactor: new metrics for resources

---
 .../src/emqx_resource_manager.erl             | 19 +++---
 .../src/emqx_resource_worker.erl              | 61 +++++++++++++------
 2 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index db4b294a8..3360c3c5d 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -129,17 +129,20 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         ?RES_METRICS,
         ResId,
         [
-            matched,
-            sent,
-            dropped,
-            queued,
-            batched,
-            inflight,
+            'matched',
+            'sent',
+            'dropped',
+            'queued',
+            'batched',
+            'retried',
             'sent.success',
             'sent.failed',
             'sent.exception',
-            'dropped.inflight',
-            'dropped.queued',
+            'sent.inflight',
+            'dropped.queue_not_enabled',
+            'dropped.queue_full',
+            'dropped.resource_not_found',
+            'dropped.resource_stopped',
             'dropped.other'
         ],
         [matched]
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 8b8b1467c..dabaf037c 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -77,27 +77,27 @@ start_link(Id, Index, Opts) ->
 sync_query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
     Timeout = maps:get(timeout, Opts, infinity),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     pick_call(Id, PickKey, {query, Request, Opts}, Timeout).
 
 -spec async_query(id(), request(), query_opts()) -> Result :: term().
 async_query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     pick_cast(Id, PickKey, {query, Request, Opts}).
 
 %% simple query the resource without batching and queuing messages.
 -spec simple_sync_query(id(), request()) -> Result :: term().
 simple_sync_query(Id, Request) ->
     Result = call_query(sync, Id, ?QUERY(self(), Request), #{}),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     _ = handle_query_result(Id, Result, false),
     Result.
 
 -spec simple_async_query(id(), request(), reply_fun()) -> Result :: term().
 simple_async_query(Id, Request, ReplyFun) ->
     Result = call_query(async, Id, ?QUERY(ReplyFun, Request), #{}),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     _ = handle_query_result(Id, Result, false),
     Result.
 
@@ -252,6 +252,7 @@ retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = S
     case handle_query_result(Id, Result, false) of
         %% Send failed because resource down
         true ->
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
             {keep_state, St0, {state_timeout, ResumeT, resume}};
         %% Send ok or failed but the resource is working
         false ->
@@ -263,18 +264,20 @@ retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = S
                         inflight_drop(Name, Ref),
                         St0;
                     _ ->
-                        St0#{queue => drop_head(Q)}
+                        St0#{queue => drop_head(Id, Q)}
                 end,
             {keep_state, St, {state_timeout, 0, resume}}
     end.
 
-drop_head(Q) ->
+drop_head(Id, Q) ->
     {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
     ok = replayq:ack(Q1, AckRef),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -1),
     Q1.
 
-query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left} = St0) ->
+query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left, id := Id} = St0) ->
     Acc1 = [?QUERY(From, Request) | Acc],
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batched'),
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
         true -> flush(St);
@@ -308,6 +311,7 @@ flush(
         inflight_name => maps:get(name, St),
         inflight_window => maps:get(async_inflight_window, St)
     },
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batched', -length(Batch)),
     Result = call_query(configured, Id, Batch, QueryOpts),
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
     case batch_reply_caller(Id, Result, Batch) of
@@ -322,8 +326,20 @@ maybe_append_queue(Id, undefined, _Items) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
     undefined;
 maybe_append_queue(Id, Q, Items) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
-    replayq:append(Q, Items).
+    case replayq:overflow(Q) of
+        Overflow when Overflow =< 0 ->
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
+            replayq:append(Q, Items);
+        Overflow ->
+            PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
+            {Q1, QAckRef, Items} = replayq:pop(Q, PopOpts),
+            ok = replayq:ack(Q1, QAckRef),
+            Dropped = length(Items),
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped),
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
+            ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
+            Q1
+    end.
 
 batch_reply_caller(Id, BatchResult, Batch) ->
     lists:foldl(
@@ -375,7 +391,8 @@ handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
 handle_query_result(Id, {error, _}, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
-handle_query_result(_Id, {recoverable_error, _}, _BlockWorker) ->
+handle_query_result(Id, {recoverable_error, _}, _BlockWorker) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', -1),
     true;
 handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
     true;
@@ -426,7 +443,7 @@ call_query(QM0, Id, Query, QueryOpts) ->
 
 apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
     ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request);
 apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
@@ -438,7 +455,8 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
                 ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
                 ReplyFun = fun ?MODULE:reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = [self(), Id, Name, Ref, Query],
@@ -451,7 +469,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
 apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
     ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent, length(Batch)),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', length(Batch)),
     ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch);
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
@@ -463,7 +481,8 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
                 ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, sent, length(Batch)),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', length(Batch)),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
                 ReplyFun = fun ?MODULE:batch_reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = {ReplyFun, [self(), Id, Name, Ref, Batch]},
@@ -477,14 +496,20 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
 
 reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request), Result) ->
     case reply_caller(Id, ?REPLY(From, Request, Result)) of
-        true -> ?MODULE:block(Pid);
-        false -> inflight_drop(Name, Ref)
+        true ->
+            ?MODULE:block(Pid);
+        false ->
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
+            inflight_drop(Name, Ref)
     end.
 
 batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
     case batch_reply_caller(Id, Result, Batch) of
-        true -> ?MODULE:block(Pid);
-        false -> inflight_drop(Name, Ref)
+        true ->
+            ?MODULE:block(Pid);
+        false ->
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -length(Batch)),
+            inflight_drop(Name, Ref)
     end.
 %%==============================================================================
 %% the inflight queue for async query

From b5ad5233a189a572a8cd98a570a73a6faac0fe15 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 10:14:56 +0800
Subject: [PATCH 107/232] fix(mqtt-bridge): username and password defaults to
 undefined

---
 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index 51144748b..f4725020e 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -101,7 +101,7 @@ fields("server_configs") ->
             mk(
                 binary(),
                 #{
-                    default => "emqx",
+                    default => undefined,
                     desc => ?DESC("username")
                 }
             )},
@@ -109,7 +109,7 @@ fields("server_configs") ->
             mk(
                 binary(),
                 #{
-                    default => "emqx",
+                    default => undefined,
                     format => <<"password">>,
                     desc => ?DESC("password")
                 }

From 262e68f7d24cd7b9335d190ebcf5d42c7dfb2a66 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 10:16:02 +0800
Subject: [PATCH 108/232] fix: return error when receive HTTP code other than
 2xx

---
 .../src/emqx_connector_http.erl               | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 5658b385d..f63873fe3 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -275,7 +275,7 @@ on_query(
     ),
     NRequest = formalize_request(Method, BasePath, Request),
     case
-        Result = ehttpc:request(
+        ehttpc:request(
             case KeyOrNum of
                 undefined -> PoolName;
                 _ -> {PoolName, KeyOrNum}
@@ -286,33 +286,35 @@ on_query(
             Retry
         )
     of
-        {error, Reason} ->
+        {error, Reason} = Result ->
             ?SLOG(error, #{
                 msg => "http_connector_do_request_failed",
                 request => NRequest,
                 reason => Reason,
                 connector => InstId
-            });
-        {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
-            ok;
-        {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
-            ok;
-        {ok, StatusCode, _} ->
+            }),
+            Result;
+        {ok, StatusCode, _} = Result when StatusCode >= 200 andalso StatusCode < 300 ->
+            Result;
+        {ok, StatusCode, _, _} = Result when StatusCode >= 200 andalso StatusCode < 300 ->
+            Result;
+        {ok, StatusCode, Headers} ->
             ?SLOG(error, #{
                 msg => "http connector do request, received error response",
                 request => NRequest,
                 connector => InstId,
                 status_code => StatusCode
-            });
-        {ok, StatusCode, _, _} ->
+            }),
+            {error, #{status_code => StatusCode, headers => Headers}};
+        {ok, StatusCode, Headers, Body} ->
             ?SLOG(error, #{
                 msg => "http connector do request, received error response",
                 request => NRequest,
                 connector => InstId,
                 status_code => StatusCode
-            })
-    end,
-    Result.
+            }),
+            {error, #{status_code => StatusCode, headers => Headers, body => Body}}
+    end.
 
 on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) ->
     case maps:get(request, State, undefined) of

From e0a6a61d739c1bec961cc4c76d3f77ceb1052960 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 10:19:40 +0800
Subject: [PATCH 109/232] fix: return error on start hstreamdb crash

---
 .../emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
index 3892b7fc0..e4bbe8425 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
@@ -135,13 +135,15 @@ start_client(InstId, Config) ->
         do_start_client(InstId, Config)
     catch
         E:R:S ->
-            ?SLOG(error, #{
+            Error = #{
                 msg => "start hstreamdb connector error",
                 connector => InstId,
                 error => E,
                 reason => R,
                 stack => S
-            })
+            },
+            ?SLOG(error, Error),
+            {error, Error}
     end.
 
 do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) ->

From 65dfa63324cb0b34ad7d6b3ae2394c4755f9d015 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 12:28:01 +0800
Subject: [PATCH 110/232] fix: update the counters for data bridges

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 195 +++++++++++++++---
 .../src/emqx_connector_http.erl               |   7 +
 2 files changed, 172 insertions(+), 30 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 36f8cf0f5..2f15bbcd7 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -57,22 +57,95 @@
     end
 ).
 
--define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{
-    matched => MATCH,
-    success => SUCC,
-    failed => FAILED,
-    rate => RATE,
-    rate_last5m => RATE_5,
-    rate_max => RATE_MAX
-}).
--define(metrics(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{
-    matched := MATCH,
-    success := SUCC,
-    failed := FAILED,
-    rate := RATE,
-    rate_last5m := RATE_5,
-    rate_max := RATE_MAX
-}).
+-define(EMPTY_METRICS,
+    ?METRICS(
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+    )
+).
+
+-define(METRICS(
+    Batched,
+    Dropped,
+    DroppedOther,
+    DroppedQueueFull,
+    DroppedQueueNotEnabled,
+    DroppedResourceNotFound,
+    DroppedResourceStopped,
+    Matched,
+    Queued,
+    Retried,
+    Sent,
+    SentExcpt,
+    SentFailed,
+    SentInflight,
+    SentSucc,
+    RATE,
+    RATE_5,
+    RATE_MAX
+),
+    #{
+        'batched' => Batched,
+        'dropped' => Dropped,
+        'dropped.other' => DroppedOther,
+        'dropped.queue_full' => DroppedQueueFull,
+        'dropped.queue_not_enabled' => DroppedQueueNotEnabled,
+        'dropped.resource_not_found' => DroppedResourceNotFound,
+        'dropped.resource_stopped' => DroppedResourceStopped,
+        'matched' => Matched,
+        'queued' => Queued,
+        'retried' => Retried,
+        'sent' => Sent,
+        'sent.exception' => SentExcpt,
+        'sent.failed' => SentFailed,
+        'sent.inflight' => SentInflight,
+        'sent.success' => SentSucc,
+        rate => RATE,
+        rate_last5m => RATE_5,
+        rate_max => RATE_MAX
+    }
+).
+
+-define(metrics(
+    Batched,
+    Dropped,
+    DroppedOther,
+    DroppedQueueFull,
+    DroppedQueueNotEnabled,
+    DroppedResourceNotFound,
+    DroppedResourceStopped,
+    Matched,
+    Queued,
+    Retried,
+    Sent,
+    SentExcpt,
+    SentFailed,
+    SentInflight,
+    SentSucc,
+    RATE,
+    RATE_5,
+    RATE_MAX
+),
+    #{
+        'batched' := Batched,
+        'dropped' := Dropped,
+        'dropped.other' := DroppedOther,
+        'dropped.queue_full' := DroppedQueueFull,
+        'dropped.queue_not_enabled' := DroppedQueueNotEnabled,
+        'dropped.resource_not_found' := DroppedResourceNotFound,
+        'dropped.resource_stopped' := DroppedResourceStopped,
+        'matched' := Matched,
+        'queued' := Queued,
+        'retried' := Retried,
+        'sent' := Sent,
+        'sent.exception' := SentExcpt,
+        'sent.failed' := SentFailed,
+        'sent.inflight' := SentInflight,
+        'sent.success' := SentSucc,
+        rate := RATE,
+        rate_last5m := RATE_5,
+        rate_max := RATE_MAX
+    }
+).
 
 namespace() -> "bridge".
 
@@ -193,11 +266,11 @@ method_example(_Type, put) ->
 
 maybe_with_metrics_example(TypeNameExam, get) ->
     TypeNameExam#{
-        metrics => ?METRICS(0, 0, 0, 0, 0, 0),
+        metrics => ?EMPTY_METRICS,
         node_metrics => [
             #{
                 node => node(),
-                metrics => ?METRICS(0, 0, 0, 0, 0, 0)
+                metrics => ?EMPTY_METRICS
             }
         ]
     };
@@ -217,7 +290,16 @@ info_example_basic(webhook) ->
         ssl => #{enable => false},
         local_topic => <<"emqx_webhook/#">>,
         method => post,
-        body => <<"${payload}">>
+        body => <<"${payload}">>,
+        resource_opts => #{
+            worker_pool_size => 1,
+            health_check_interval => 15000,
+            auto_restart_interval => 15000,
+            query_mode => sync,
+            async_inflight_window => 100,
+            enable_queue => true,
+            max_queue_bytes => 1024 * 1024 * 1024
+        }
     };
 info_example_basic(mqtt) ->
     (mqtt_main_example())#{
@@ -619,19 +701,37 @@ collect_metrics(Bridges) ->
     [maps:with([node, metrics], B) || B <- Bridges].
 
 aggregate_metrics(AllMetrics) ->
-    InitMetrics = ?METRICS(0, 0, 0, 0, 0, 0),
+    InitMetrics = ?EMPTY_METRICS,
     lists:foldl(
         fun(
-            #{metrics := ?metrics(Match1, Succ1, Failed1, Rate1, Rate5m1, RateMax1)},
-            ?metrics(Match0, Succ0, Failed0, Rate0, Rate5m0, RateMax0)
+            #{
+                metrics := ?metrics(
+                    M1, M2, M3, M4, M5, M6, M7, M8, M9, M10, M11, M12, M13, M14, M15, M16, M17, M18
+                )
+            },
+            ?metrics(
+                N1, N2, N3, N4, N5, N6, N7, N8, N9, N10, N11, N12, N13, N14, N15, N16, N17, N18
+            )
         ) ->
             ?METRICS(
-                Match1 + Match0,
-                Succ1 + Succ0,
-                Failed1 + Failed0,
-                Rate1 + Rate0,
-                Rate5m1 + Rate5m0,
-                RateMax1 + RateMax0
+                M1 + N1,
+                M2 + N2,
+                M3 + N3,
+                M4 + N4,
+                M5 + N5,
+                M6 + N6,
+                M7 + N7,
+                M8 + N8,
+                M9 + N9,
+                M10 + N10,
+                M11 + N11,
+                M12 + N12,
+                M13 + N13,
+                M14 + N14,
+                M15 + N15,
+                M16 + N16,
+                M17 + N17,
+                M18 + N18
             )
         end,
         InitMetrics,
@@ -660,12 +760,47 @@ format_resp(
     }.
 
 format_metrics(#{
-    counters := #{failed := Failed, exception := Ex, matched := Match, success := Succ},
+    counters := #{
+        'batched' := Batched,
+        'dropped' := Dropped,
+        'dropped.other' := DroppedOther,
+        'dropped.queue_full' := DroppedQueueFull,
+        'dropped.queue_not_enabled' := DroppedQueueNotEnabled,
+        'dropped.resource_not_found' := DroppedResourceNotFound,
+        'dropped.resource_stopped' := DroppedResourceStopped,
+        'matched' := Matched,
+        'queued' := Queued,
+        'retried' := Retried,
+        'sent' := Sent,
+        'sent.exception' := SentExcpt,
+        'sent.failed' := SentFailed,
+        'sent.inflight' := SentInflight,
+        'sent.success' := SentSucc
+    },
     rate := #{
         matched := #{current := Rate, last5m := Rate5m, max := RateMax}
     }
 }) ->
-    ?METRICS(Match, Succ, Failed + Ex, Rate, Rate5m, RateMax).
+    ?METRICS(
+        Batched,
+        Dropped,
+        DroppedOther,
+        DroppedQueueFull,
+        DroppedQueueNotEnabled,
+        DroppedResourceNotFound,
+        DroppedResourceStopped,
+        Matched,
+        Queued,
+        Retried,
+        Sent,
+        SentExcpt,
+        SentFailed,
+        SentInflight,
+        SentSucc,
+        Rate,
+        Rate5m,
+        RateMax
+    ).
 
 fill_defaults(Type, RawConf) ->
     PackedConf = pack_bridge_conf(Type, RawConf),
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index f63873fe3..2562bf272 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -286,6 +286,13 @@ on_query(
             Retry
         )
     of
+        {error, econnrefused} ->
+            ?SLOG(warning, #{
+                msg => "http_connector_do_request_failed",
+                reason => econnrefused,
+                connector => InstId
+            }),
+            {recoverable_error, econnrefused};
         {error, Reason} = Result ->
             ?SLOG(error, #{
                 msg => "http_connector_do_request_failed",

From c4106c0d7751cecf9dc5765131682fd348a0da34 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 12:28:43 +0800
Subject: [PATCH 111/232] fix: resume the resource worker on health check
 success

---
 apps/emqx_resource/src/emqx_resource_manager.erl    | 7 +++++++
 apps/emqx_resource/src/emqx_resource_worker_sup.erl | 6 +++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 3360c3c5d..261863d4c 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -555,6 +555,7 @@ with_health_check(Data, Func) ->
     HCRes = emqx_resource:call_health_check(Data#data.manager_id, Data#data.mod, Data#data.state),
     {Status, NewState, Err} = parse_health_check_result(HCRes, Data),
     _ = maybe_alarm(Status, ResId),
+    ok = maybe_resume_resource_workers(Status),
     UpdatedData = Data#data{
         state = NewState, status = Status, error = Err
     },
@@ -575,6 +576,12 @@ maybe_alarm(_Status, ResId) ->
         <<"resource down: ", ResId/binary>>
     ).
 
+maybe_resume_resource_workers(connected) ->
+    {_, Pid, _, _} = supervisor:which_children(emqx_resource_worker_sup),
+    emqx_resource_worker:resume(Pid);
+maybe_resume_resource_workers(_) ->
+    ok.
+
 maybe_clear_alarm(<<?TEST_ID_PREFIX, _/binary>>) ->
     ok;
 maybe_clear_alarm(ResId) ->
diff --git a/apps/emqx_resource/src/emqx_resource_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_worker_sup.erl
index 5305eddaf..2db7b5c4c 100644
--- a/apps/emqx_resource/src/emqx_resource_worker_sup.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker_sup.erl
@@ -107,7 +107,7 @@ ensure_worker_started(ResId, Idx, Opts) ->
         type => worker,
         modules => [Mod]
     },
-    case supervisor:start_child(emqx_resource_sup, Spec) of
+    case supervisor:start_child(?SERVER, Spec) of
         {ok, _Pid} -> ok;
         {error, {already_started, _}} -> ok;
         {error, already_present} -> ok;
@@ -116,9 +116,9 @@ ensure_worker_started(ResId, Idx, Opts) ->
 
 ensure_worker_removed(ResId, Idx) ->
     ChildId = ?CHILD_ID(emqx_resource_worker, ResId, Idx),
-    case supervisor:terminate_child(emqx_resource_sup, ChildId) of
+    case supervisor:terminate_child(?SERVER, ChildId) of
         ok ->
-            Res = supervisor:delete_child(emqx_resource_sup, ChildId),
+            Res = supervisor:delete_child(?SERVER, ChildId),
             _ = gproc_pool:remove_worker(ResId, {ResId, Idx}),
             Res;
         {error, not_found} ->

From 9e50866cd0ab7d5d5682c4b06cf7df97dbc10117 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 17:13:49 +0800
Subject: [PATCH 112/232] fix: rename queue_max_bytes -> max_queue_bytes

---
 apps/emqx/src/emqx_metrics_worker.erl         |  2 +-
 .../i18n/emqx_resource_schema_i18n.conf       |  2 +-
 apps/emqx_resource/include/emqx_resource.hrl  |  9 +++--
 .../src/emqx_resource_manager.erl             |  8 +++--
 .../src/emqx_resource_worker.erl              | 36 +++++++++++--------
 .../src/schema/emqx_resource_schema.erl       | 12 +++----
 .../src/emqx_rule_runtime.erl                 |  2 +-
 7 files changed, 42 insertions(+), 29 deletions(-)

diff --git a/apps/emqx/src/emqx_metrics_worker.erl b/apps/emqx/src/emqx_metrics_worker.erl
index 21e73ff51..ab6a0b1a6 100644
--- a/apps/emqx/src/emqx_metrics_worker.erl
+++ b/apps/emqx/src/emqx_metrics_worker.erl
@@ -173,7 +173,7 @@ get_metrics(Name, Id) ->
 inc(Name, Id, Metric) ->
     inc(Name, Id, Metric, 1).
 
--spec inc(handler_name(), metric_id(), atom(), pos_integer()) -> ok.
+-spec inc(handler_name(), metric_id(), atom(), integer()) -> ok.
 inc(Name, Id, Metric, Val) ->
     counters:add(get_ref(Name, Id), idx_metric(Name, Id, Metric), Val).
 
diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index 0d53b813e..d7953ac3b 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -143,7 +143,7 @@ emqx_resource_schema {
     }
   }
 
-  queue_max_bytes {
+  max_queue_bytes {
     desc {
       en: """Maximum queue storage."""
       zh: """消息队列的最大长度。"""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 4d1c45eb4..2409a7069 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -68,7 +68,7 @@
     batch_size => pos_integer(),
     batch_time => pos_integer(),
     enable_queue => boolean(),
-    queue_max_bytes => pos_integer(),
+    max_queue_bytes => pos_integer(),
     query_mode => query_mode(),
     resume_interval => pos_integer(),
     async_inflight_window => pos_integer()
@@ -81,8 +81,11 @@
 
 -define(WORKER_POOL_SIZE, 16).
 
--define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
--define(DEFAULT_QUEUE_SIZE_RAW, <<"1GB">>).
+-define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024).
+-define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>).
+
+-define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024 * 1024).
+-define(DEFAULT_QUEUE_SIZE_RAW, <<"100GB">>).
 
 %% count
 -define(DEFAULT_BATCH_SIZE, 100).
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 261863d4c..eaf0a9a5f 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -577,8 +577,12 @@ maybe_alarm(_Status, ResId) ->
     ).
 
 maybe_resume_resource_workers(connected) ->
-    {_, Pid, _, _} = supervisor:which_children(emqx_resource_worker_sup),
-    emqx_resource_worker:resume(Pid);
+    lists:foreach(
+        fun({_, Pid, _, _}) ->
+            emqx_resource_worker:resume(Pid)
+        end,
+        supervisor:which_children(emqx_resource_worker_sup)
+    );
 maybe_resume_resource_workers(_) ->
     ok.
 
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index dabaf037c..013f430b1 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -123,13 +123,15 @@ init({Id, Index, Opts}) ->
             true ->
                 replayq:open(#{
                     dir => disk_queue_dir(Id, Index),
-                    seg_bytes => maps:get(queue_max_bytes, Opts, ?DEFAULT_QUEUE_SIZE),
+                    seg_bytes => maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE),
+                    max_total_bytes => maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE),
                     sizer => fun ?MODULE:estimate_size/1,
                     marshaller => fun ?MODULE:queue_item_marshaller/1
                 });
             false ->
                 undefined
         end,
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', replayq:count(Queue)),
     ok = inflight_new(Name),
     St = #{
         id => Id,
@@ -323,23 +325,27 @@ flush(
     end.
 
 maybe_append_queue(Id, undefined, _Items) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
     undefined;
 maybe_append_queue(Id, Q, Items) ->
-    case replayq:overflow(Q) of
-        Overflow when Overflow =< 0 ->
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
-            replayq:append(Q, Items);
-        Overflow ->
-            PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
-            {Q1, QAckRef, Items} = replayq:pop(Q, PopOpts),
-            ok = replayq:ack(Q1, QAckRef),
-            Dropped = length(Items),
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped),
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
-            ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
-            Q1
-    end.
+    Q2 =
+        case replayq:overflow(Q) of
+            Overflow when Overflow =< 0 ->
+                Q;
+            Overflow ->
+                PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
+                {Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts),
+                ok = replayq:ack(Q1, QAckRef),
+                Dropped = length(Items2),
+                emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped),
+                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
+                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
+                ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
+                Q1
+        end,
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
+    replayq:append(Q2, Items).
 
 batch_reply_caller(Id, BatchResult, Batch) ->
     lists:foldl(
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index fe8564a41..9e54c8a7b 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -53,7 +53,7 @@ fields("creation_opts") ->
         {batch_size, fun batch_size/1},
         {batch_time, fun batch_time/1},
         {enable_queue, fun enable_queue/1},
-        {max_queue_bytes, fun queue_max_bytes/1}
+        {max_queue_bytes, fun max_queue_bytes/1}
     ].
 
 worker_pool_size(type) -> pos_integer();
@@ -110,11 +110,11 @@ batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW;
 batch_time(required) -> false;
 batch_time(_) -> undefined.
 
-queue_max_bytes(type) -> emqx_schema:bytesize();
-queue_max_bytes(desc) -> ?DESC("queue_max_bytes");
-queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW;
-queue_max_bytes(required) -> false;
-queue_max_bytes(_) -> undefined.
+max_queue_bytes(type) -> emqx_schema:bytesize();
+max_queue_bytes(desc) -> ?DESC("max_queue_bytes");
+max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW;
+max_queue_bytes(required) -> false;
+max_queue_bytes(_) -> undefined.
 
 desc("creation_opts") ->
     ?DESC("creation_opts").
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index aafce137c..847c4ff00 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -522,6 +522,6 @@ inc_action_metrics(R, RuleId) ->
 is_ok_result(ok) ->
     true;
 is_ok_result(R) when is_tuple(R) ->
-    ok = erlang:element(1, R);
+    ok == erlang:element(1, R);
 is_ok_result(ok) ->
     false.

From 73e19d84ee2c169f6783a8bcd2de9450234adbb3 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 30 Aug 2022 23:47:58 +0800
Subject: [PATCH 113/232] feat: use the new metrics to bridge APIs

---
 apps/emqx_bridge/i18n/emqx_bridge_schema.conf | 152 ++++++++++++++++--
 apps/emqx_bridge/include/emqx_bridge.hrl      |  89 ++++++++++
 apps/emqx_bridge/src/emqx_bridge_api.erl      |  99 +-----------
 .../src/schema/emqx_bridge_schema.erl         |  21 ++-
 .../test/emqx_bridge_mqtt_SUITE.erl           |  64 ++++++--
 .../src/emqx_connector_mqtt.erl               |   3 +-
 apps/emqx_resource/src/emqx_resource.erl      |  17 +-
 .../src/emqx_resource_manager.erl             |   4 +-
 .../src/emqx_resource_worker.erl              |   8 +-
 .../test/emqx_resource_SUITE.erl              |   8 +-
 .../src/emqx_rule_sqltester.erl               |  18 ++-
 11 files changed, 327 insertions(+), 156 deletions(-)
 create mode 100644 apps/emqx_bridge/include/emqx_bridge.hrl

diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
index 704fd7bd7..06cc41a91 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
@@ -78,36 +78,149 @@ emqx_bridge_schema {
                           }
                   }
 
+    metric_batched {
+                   desc {
+                         en: """Count of messages that are currently accumulated in memory waiting for sending in one batch."""
+                         zh: """当前积压在内存里，等待批量发送的消息个数"""
+                        }
+                   label: {
+                           en: "Batched"
+                           zh: "等待批量发送"
+                          }
+                  }
+
+    metric_dropped {
+                   desc {
+                         en: """Count of messages dropped."""
+                         zh: """被丢弃的消息个数。"""
+                        }
+                   label: {
+                           en: "Dropped"
+                           zh: "丢弃"
+                          }
+                  }
+
+    metric_dropped_other {
+                   desc {
+                         en: """Count of messages dropped due to other reasons."""
+                         zh: """因为其他原因被丢弃的消息个数。"""
+                        }
+                   label: {
+                           en: "Dropped Other"
+                           zh: "其他丢弃"
+                          }
+                  }
+    metric_dropped_queue_full {
+                   desc {
+                         en: """Count of messages dropped due to the queue is full."""
+                         zh: """因为队列已满被丢弃的消息个数。"""
+                        }
+                   label: {
+                           en: "Dropped Queue Full"
+                           zh: "队列已满被丢弃"
+                          }
+                  }
+    metric_dropped_queue_not_enabled {
+                   desc {
+                         en: """Count of messages dropped due to the queue is not enabled."""
+                         zh: """因为队列未启用被丢弃的消息个数。"""
+                        }
+                   label: {
+                           en: "Dropped Queue Disabled"
+                           zh: "队列未启用被丢弃"
+                          }
+                  }
+    metric_dropped_resource_not_found {
+                   desc {
+                         en: """Count of messages dropped due to the resource is not found."""
+                         zh: """因为资源不存在被丢弃的消息个数。"""
+                        }
+                   label: {
+                           en: "Dropped Resource NotFound"
+                           zh: "资源不存在被丢弃"
+                          }
+                  }
+    metric_dropped_resource_stopped {
+                   desc {
+                         en: """Count of messages dropped due to the resource is stopped."""
+                         zh: """因为资源已停用被丢弃的消息个数。"""
+                        }
+                   label: {
+                           en: "Dropped Resource Stopped"
+                           zh: "资源停用被丢弃"
+                          }
+                  }
     metric_matched {
                    desc {
-                         en: """Count of this bridge is queried"""
-                         zh: """Bridge 执行操作的次数"""
+                         en: """Count of this bridge is matched and queried."""
+                         zh: """Bridge 被匹配到（被请求）的次数。"""
                         }
                    label: {
-                           en: "Bridge Matched"
-                           zh: "Bridge 执行操作的次数"
+                           en: "Matched"
+                           zh: "匹配次数"
                           }
                   }
 
-    metric_success {
+     metric_queued {
                    desc {
-                         en: """Count of query success"""
-                         zh: """Bridge 执行操作成功的次数"""
+                         en: """Count of messages that are currently queued."""
+                         zh: """当前被缓存到磁盘队列的消息个数。"""
                         }
                    label: {
-                           en: "Bridge Success"
-                           zh: "Bridge 执行操作成功的次数"
+                           en: "Queued"
+                           zh: "被缓存"
+                          }
+                  }
+     metric_sent {
+                   desc {
+                         en: """Count of messages that are sent by this bridge."""
+                         zh: """已经发送出去的消息个数。"""
+                        }
+                   label: {
+                           en: "Sent"
+                           zh: "已发送"
+                          }
+                  }
+     metric_sent_exception {
+                   desc {
+                         en: """Count of messages that were sent but exceptions occur."""
+                         zh: """发送出现异常的消息个数。"""
+                        }
+                   label: {
+                           en: "Sent Exception"
+                           zh: "发送异常"
                           }
                   }
 
-    metric_failed {
+     metric_sent_failed {
                    desc {
-                         en: """Count of query failed"""
-                         zh: """Bridge 执行操作失败的次数"""
+                         en: """Count of messages that sent failed."""
+                         zh: """发送失败的消息个数。"""
                         }
                    label: {
-                           en: "Bridge Failed"
-                           zh: "Bridge 执行操作失败的次数"
+                           en: "Sent Failed"
+                           zh: "发送失败"
+                          }
+                  }
+
+    metric_sent_inflight {
+                   desc {
+                         en: """Count of messages that were sent asynchronously but ACKs are not received."""
+                         zh: """已异步地发送但没有收到 ACK 的消息个数。"""
+                        }
+                   label: {
+                           en: "Sent Inflight"
+                           zh: "已发送未确认"
+                          }
+                  }
+    metric_sent_success {
+                   desc {
+                         en: """Count of messages that sent successfully."""
+                         zh: """已经发送成功的消息个数。"""
+                        }
+                   label: {
+                           en: "Sent Success"
+                           zh: "发送成功"
                           }
                   }
 
@@ -144,6 +257,17 @@ emqx_bridge_schema {
                           }
                   }
 
+    metric_received {
+                   desc {
+                         en: """Count of messages that is received from the remote system."""
+                         zh: """从远程系统收到的消息个数。"""
+                        }
+                   label: {
+                           en: "Received"
+                           zh: "已接收"
+                          }
+                  }
+
     desc_bridges {
                    desc {
                          en: """Configuration for MQTT bridges."""
diff --git a/apps/emqx_bridge/include/emqx_bridge.hrl b/apps/emqx_bridge/include/emqx_bridge.hrl
new file mode 100644
index 000000000..217c403b9
--- /dev/null
+++ b/apps/emqx_bridge/include/emqx_bridge.hrl
@@ -0,0 +1,89 @@
+-define(EMPTY_METRICS,
+    ?METRICS(
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+    )
+).
+
+-define(METRICS(
+    Batched,
+    Dropped,
+    DroppedOther,
+    DroppedQueueFull,
+    DroppedQueueNotEnabled,
+    DroppedResourceNotFound,
+    DroppedResourceStopped,
+    Matched,
+    Queued,
+    Sent,
+    SentExcpt,
+    SentFailed,
+    SentInflight,
+    SentSucc,
+    RATE,
+    RATE_5,
+    RATE_MAX,
+    Rcvd
+),
+    #{
+        'batched' => Batched,
+        'dropped' => Dropped,
+        'dropped.other' => DroppedOther,
+        'dropped.queue_full' => DroppedQueueFull,
+        'dropped.queue_not_enabled' => DroppedQueueNotEnabled,
+        'dropped.resource_not_found' => DroppedResourceNotFound,
+        'dropped.resource_stopped' => DroppedResourceStopped,
+        'matched' => Matched,
+        'queued' => Queued,
+        'sent' => Sent,
+        'sent.exception' => SentExcpt,
+        'sent.failed' => SentFailed,
+        'sent.inflight' => SentInflight,
+        'sent.success' => SentSucc,
+        rate => RATE,
+        rate_last5m => RATE_5,
+        rate_max => RATE_MAX,
+        received => Rcvd
+    }
+).
+
+-define(metrics(
+    Batched,
+    Dropped,
+    DroppedOther,
+    DroppedQueueFull,
+    DroppedQueueNotEnabled,
+    DroppedResourceNotFound,
+    DroppedResourceStopped,
+    Matched,
+    Queued,
+    Sent,
+    SentExcpt,
+    SentFailed,
+    SentInflight,
+    SentSucc,
+    RATE,
+    RATE_5,
+    RATE_MAX,
+    Rcvd
+),
+    #{
+        'batched' := Batched,
+        'dropped' := Dropped,
+        'dropped.other' := DroppedOther,
+        'dropped.queue_full' := DroppedQueueFull,
+        'dropped.queue_not_enabled' := DroppedQueueNotEnabled,
+        'dropped.resource_not_found' := DroppedResourceNotFound,
+        'dropped.resource_stopped' := DroppedResourceStopped,
+        'matched' := Matched,
+        'queued' := Queued,
+        'sent' := Sent,
+        'sent.exception' := SentExcpt,
+        'sent.failed' := SentFailed,
+        'sent.inflight' := SentInflight,
+        'sent.success' := SentSucc,
+        rate := RATE,
+        rate_last5m := RATE_5,
+        rate_max := RATE_MAX,
+        received := Rcvd
+    }
+).
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 2f15bbcd7..ddffcb79f 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -20,6 +20,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
 
 -import(hoconsc, [mk/2, array/1, enum/1]).
 
@@ -57,96 +58,6 @@
     end
 ).
 
--define(EMPTY_METRICS,
-    ?METRICS(
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-    )
-).
-
--define(METRICS(
-    Batched,
-    Dropped,
-    DroppedOther,
-    DroppedQueueFull,
-    DroppedQueueNotEnabled,
-    DroppedResourceNotFound,
-    DroppedResourceStopped,
-    Matched,
-    Queued,
-    Retried,
-    Sent,
-    SentExcpt,
-    SentFailed,
-    SentInflight,
-    SentSucc,
-    RATE,
-    RATE_5,
-    RATE_MAX
-),
-    #{
-        'batched' => Batched,
-        'dropped' => Dropped,
-        'dropped.other' => DroppedOther,
-        'dropped.queue_full' => DroppedQueueFull,
-        'dropped.queue_not_enabled' => DroppedQueueNotEnabled,
-        'dropped.resource_not_found' => DroppedResourceNotFound,
-        'dropped.resource_stopped' => DroppedResourceStopped,
-        'matched' => Matched,
-        'queued' => Queued,
-        'retried' => Retried,
-        'sent' => Sent,
-        'sent.exception' => SentExcpt,
-        'sent.failed' => SentFailed,
-        'sent.inflight' => SentInflight,
-        'sent.success' => SentSucc,
-        rate => RATE,
-        rate_last5m => RATE_5,
-        rate_max => RATE_MAX
-    }
-).
-
--define(metrics(
-    Batched,
-    Dropped,
-    DroppedOther,
-    DroppedQueueFull,
-    DroppedQueueNotEnabled,
-    DroppedResourceNotFound,
-    DroppedResourceStopped,
-    Matched,
-    Queued,
-    Retried,
-    Sent,
-    SentExcpt,
-    SentFailed,
-    SentInflight,
-    SentSucc,
-    RATE,
-    RATE_5,
-    RATE_MAX
-),
-    #{
-        'batched' := Batched,
-        'dropped' := Dropped,
-        'dropped.other' := DroppedOther,
-        'dropped.queue_full' := DroppedQueueFull,
-        'dropped.queue_not_enabled' := DroppedQueueNotEnabled,
-        'dropped.resource_not_found' := DroppedResourceNotFound,
-        'dropped.resource_stopped' := DroppedResourceStopped,
-        'matched' := Matched,
-        'queued' := Queued,
-        'retried' := Retried,
-        'sent' := Sent,
-        'sent.exception' := SentExcpt,
-        'sent.failed' := SentFailed,
-        'sent.inflight' := SentInflight,
-        'sent.success' := SentSucc,
-        rate := RATE,
-        rate_last5m := RATE_5,
-        rate_max := RATE_MAX
-    }
-).
-
 namespace() -> "bridge".
 
 api_spec() ->
@@ -770,12 +681,12 @@ format_metrics(#{
         'dropped.resource_stopped' := DroppedResourceStopped,
         'matched' := Matched,
         'queued' := Queued,
-        'retried' := Retried,
         'sent' := Sent,
         'sent.exception' := SentExcpt,
         'sent.failed' := SentFailed,
         'sent.inflight' := SentInflight,
-        'sent.success' := SentSucc
+        'sent.success' := SentSucc,
+        'received' := Rcvd
     },
     rate := #{
         matched := #{current := Rate, last5m := Rate5m, max := RateMax}
@@ -791,7 +702,6 @@ format_metrics(#{
         DroppedResourceStopped,
         Matched,
         Queued,
-        Retried,
         Sent,
         SentExcpt,
         SentFailed,
@@ -799,7 +709,8 @@ format_metrics(#{
         SentSucc,
         Rate,
         Rate5m,
-        RateMax
+        RateMax,
+        Rcvd
     ).
 
 fill_defaults(Type, RawConf) ->
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index aedfcaa03..f55ac840e 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -102,16 +102,31 @@ fields(bridges) ->
     ] ++ ee_fields_bridges();
 fields("metrics") ->
     [
+        {"batched", mk(integer(), #{desc => ?DESC("metric_batched")})},
+        {"dropped", mk(integer(), #{desc => ?DESC("metric_dropped")})},
+        {"dropped.other", mk(integer(), #{desc => ?DESC("metric_dropped_other")})},
+        {"dropped.queue_full", mk(integer(), #{desc => ?DESC("metric_dropped_queue_full")})},
+        {"dropped.queue_not_enabled",
+            mk(integer(), #{desc => ?DESC("metric_dropped_queue_not_enabled")})},
+        {"dropped.resource_not_found",
+            mk(integer(), #{desc => ?DESC("metric_dropped_resource_not_found")})},
+        {"dropped.resource_stopped",
+            mk(integer(), #{desc => ?DESC("metric_dropped_resource_stopped")})},
         {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})},
-        {"success", mk(integer(), #{desc => ?DESC("metric_success")})},
-        {"failed", mk(integer(), #{desc => ?DESC("metric_failed")})},
+        {"queued", mk(integer(), #{desc => ?DESC("metric_queued")})},
+        {"sent", mk(integer(), #{desc => ?DESC("metric_sent")})},
+        {"sent.exception", mk(integer(), #{desc => ?DESC("metric_sent_exception")})},
+        {"sent.failed", mk(integer(), #{desc => ?DESC("metric_sent_failed")})},
+        {"sent.inflight", mk(integer(), #{desc => ?DESC("metric_sent_inflight")})},
+        {"sent.success", mk(integer(), #{desc => ?DESC("metric_sent_success")})},
         {"rate", mk(float(), #{desc => ?DESC("metric_rate")})},
         {"rate_max", mk(float(), #{desc => ?DESC("metric_rate_max")})},
         {"rate_last5m",
             mk(
                 float(),
                 #{desc => ?DESC("metric_rate_last5m")}
-            )}
+            )},
+        {"received", mk(float(), #{desc => ?DESC("metric_received")})}
     ];
 fields("node_metrics") ->
     [
diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
index 27c101728..02b76d64b 100644
--- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -66,15 +66,6 @@
     }
 }).
 
--define(metrics(MATCH, SUCC, FAILED, SPEED, SPEED5M, SPEEDMAX), #{
-    <<"matched">> := MATCH,
-    <<"success">> := SUCC,
-    <<"failed">> := FAILED,
-    <<"rate">> := SPEED,
-    <<"rate_last5m">> := SPEED5M,
-    <<"rate_max">> := SPEEDMAX
-}).
-
 inspect(Selected, _Envs, _Args) ->
     persistent_term:put(?MODULE, #{inspect => Selected}).
 
@@ -185,6 +176,23 @@ t_mqtt_conn_bridge_ingress(_) ->
         end
     ),
 
+    %% verify the metrics of the bridge
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDIngress]), []),
+    ?assertMatch(
+        #{
+            <<"metrics">> := #{<<"matched">> := 0, <<"received">> := 1},
+            <<"node_metrics">> :=
+                [
+                    #{
+                        <<"node">> := _,
+                        <<"metrics">> :=
+                            #{<<"matched">> := 0, <<"received">> := 1}
+                    }
+                ]
+        },
+        jsx:decode(BridgeStr)
+    ),
+
     %% delete the bridge
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
@@ -237,9 +245,15 @@ t_mqtt_conn_bridge_egress(_) ->
     {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
     ?assertMatch(
         #{
-            <<"metrics">> := ?metrics(1, 1, 0, _, _, _),
+            <<"metrics">> := #{<<"matched">> := 1, <<"sent.success">> := 1, <<"sent.failed">> := 0},
             <<"node_metrics">> :=
-                [#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}]
+                [
+                    #{
+                        <<"node">> := _,
+                        <<"metrics">> :=
+                            #{<<"matched">> := 1, <<"sent.success">> := 1, <<"sent.failed">> := 0}
+                    }
+                ]
         },
         jsx:decode(BridgeStr)
     ),
@@ -337,6 +351,23 @@ t_ingress_mqtt_bridge_with_rules(_) ->
         persistent_term:get(?MODULE)
     ),
 
+    %% verify the metrics of the bridge
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDIngress]), []),
+    ?assertMatch(
+        #{
+            <<"metrics">> := #{<<"matched">> := 0, <<"received">> := 1},
+            <<"node_metrics">> :=
+                [
+                    #{
+                        <<"node">> := _,
+                        <<"metrics">> :=
+                            #{<<"matched">> := 0, <<"received">> := 1}
+                    }
+                ]
+        },
+        jsx:decode(BridgeStr)
+    ),
+
     {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []).
 
@@ -433,9 +464,16 @@ t_egress_mqtt_bridge_with_rules(_) ->
     {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
     ?assertMatch(
         #{
-            <<"metrics">> := ?metrics(2, 2, 0, _, _, _),
+            <<"metrics">> := #{<<"matched">> := 2, <<"sent.success">> := 2, <<"sent.failed">> := 0},
             <<"node_metrics">> :=
-                [#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}]
+                [
+                    #{
+                        <<"node">> := _,
+                        <<"metrics">> := #{
+                            <<"matched">> := 2, <<"sent.success">> := 2, <<"sent.failed">> := 0
+                        }
+                    }
+                ]
         },
         jsx:decode(BridgeStr)
     ),
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index bdf43885a..3ce6925bc 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -135,8 +135,7 @@ drop_bridge(Name) ->
 %% When use this bridge as a data source, ?MODULE:on_message_received will be called
 %% if the bridge received msgs from the remote broker.
 on_message_received(Msg, HookPoint, ResId) ->
-    emqx_resource:inc_matched(ResId),
-    emqx_resource:inc_success(ResId),
+    emqx_resource:inc_received(ResId),
     emqx:run_hook(HookPoint, [Msg]).
 
 %% ===================================================================
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index b07460643..c1d500e8b 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -110,7 +110,7 @@
     list_group_instances/1
 ]).
 
--export([inc_metrics_funcs/1, inc_matched/1, inc_success/1, inc_failed/1]).
+-export([inc_received/1]).
 
 -optional_callbacks([
     on_query/3,
@@ -443,19 +443,8 @@ check_and_do(ResourceType, RawConfig, Do) when is_function(Do) ->
 
 %% =================================================================================
 
-inc_matched(ResId) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ResId, matched).
-
-inc_success(ResId) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ResId, success).
-
-inc_failed(ResId) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ResId, failed).
+inc_received(ResId) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ResId, 'received').
 
 filter_instances(Filter) ->
     [Id || #{id := Id, mod := Mod} <- list_instances_verbose(), Filter(Id, Mod)].
-
-inc_metrics_funcs(ResId) ->
-    OnSucc = [{fun ?MODULE:inc_success/1, ResId}],
-    OnFailed = [{fun ?MODULE:inc_failed/1, ResId}],
-    {OnSucc, OnFailed}.
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index eaf0a9a5f..2581a3001 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -134,7 +134,6 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
             'dropped',
             'queued',
             'batched',
-            'retried',
             'sent.success',
             'sent.failed',
             'sent.exception',
@@ -143,7 +142,8 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
             'dropped.queue_full',
             'dropped.resource_not_found',
             'dropped.resource_stopped',
-            'dropped.other'
+            'dropped.other',
+            'received'
         ],
         [matched]
     ),
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 013f430b1..3a12a6b91 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -131,7 +131,7 @@ init({Id, Index, Opts}) ->
             false ->
                 undefined
         end,
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', replayq:count(Queue)),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', queue_count(Queue)),
     ok = inflight_new(Name),
     St = #{
         id => Id,
@@ -254,7 +254,6 @@ retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = S
     case handle_query_result(Id, Result, false) of
         %% Send failed because resource down
         true ->
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
             {keep_state, St0, {state_timeout, ResumeT, resume}};
         %% Send ok or failed but the resource is working
         false ->
@@ -569,6 +568,11 @@ assert_ok_result(R) when is_tuple(R) ->
 assert_ok_result(R) ->
     error({not_ok_result, R}).
 
+queue_count(undefined) ->
+    0;
+queue_count(Q) ->
+    replayq:count(Q).
+
 -spec name(id(), integer()) -> atom().
 name(Id, Index) ->
     Mod = atom_to_list(?MODULE),
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 0bfa67d07..68b4fb6dd 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -268,7 +268,7 @@ t_query_counter_async_query(_) ->
         end
     ),
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
-    ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C),
+    ?assertMatch(#{matched := 1002, 'sent.success' := 1002, 'sent.failed' := 0}, C),
     ok = emqx_resource:remove_local(?ID).
 
 t_query_counter_async_callback(_) ->
@@ -309,7 +309,7 @@ t_query_counter_async_callback(_) ->
         end
     ),
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
-    ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C),
+    ?assertMatch(#{matched := 1002, sent := 1002, 'sent.success' := 1002, 'sent.failed' := 0}, C),
     ?assertMatch(1000, ets:info(Tab0, size)),
     ?assert(
         lists:all(
@@ -419,8 +419,8 @@ t_query_counter_async_inflight(_) ->
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
     ct:pal("metrics: ~p", [C]),
     ?assertMatch(
-        #{matched := M, success := S, exception := E, failed := F, recoverable_error := RD} when
-            M >= Sent andalso M == S + E + F + RD,
+        #{matched := M, sent := St, 'sent.success' := Ss, dropped := D} when
+            St == Ss andalso M == St + D,
         C
     ),
     ?assert(
diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
index c333bb80e..9669e0113 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
@@ -62,14 +62,16 @@ test_rule(Sql, Select, Context, EventTopics) ->
     },
     FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)),
     try emqx_rule_runtime:apply_rule(Rule, FullContext, #{}) of
-        {ok, Data} -> {ok, flatten(Data)};
-        {error, Reason} -> {error, Reason}
+        {ok, Data} ->
+            {ok, flatten(Data)};
+        {error, Reason} ->
+            {error, Reason}
     after
         ok = emqx_rule_engine:clear_metrics_for_rule(RuleId)
     end.
 
 get_selected_data(Selected, _Envs, _Args) ->
-    Selected.
+    {ok, Selected}.
 
 is_publish_topic(<<"$events/", _/binary>>) -> false;
 is_publish_topic(<<"$bridges/", _/binary>>) -> false;
@@ -77,14 +79,14 @@ is_publish_topic(_Topic) -> true.
 
 flatten([]) ->
     [];
-flatten([D1]) ->
-    D1;
-flatten([D1 | L]) when is_list(D1) ->
-    D1 ++ flatten(L).
+flatten([{ok, D}]) ->
+    D;
+flatten([D | L]) when is_list(D) ->
+    [D0 || {ok, D0} <- D] ++ flatten(L).
 
 echo_action(Data, Envs) ->
     ?TRACE("RULE", "testing_rule_sql_ok", #{data => Data, envs => Envs}),
-    Data.
+    {ok, Data}.
 
 fill_default_values(Event, Context) ->
     maps:merge(envs_examp(Event), Context).

From ca52b8eb297b0f128c3b6305d4ff79b1350eee76 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 31 Aug 2022 09:18:10 +0800
Subject: [PATCH 114/232] fix: start connector-mqtt failed when
 username/password not provided

---
 .../src/emqx_connector_mqtt.erl               | 43 ++++++++++++-------
 apps/emqx_rule_engine/include/rule_engine.hrl | 14 +++---
 .../src/emqx_rule_runtime.erl                 | 12 +++---
 3 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 3ce6925bc..915fd8007 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -224,20 +224,20 @@ make_forward_confs(undefined) ->
 make_forward_confs(FrowardConf) ->
     FrowardConf.
 
-basic_config(#{
-    server := Server,
-    reconnect_interval := ReconnIntv,
-    proto_ver := ProtoVer,
-    bridge_mode := BridgeMode,
-    username := User,
-    password := Password,
-    clean_start := CleanStart,
-    keepalive := KeepAlive,
-    retry_interval := RetryIntv,
-    max_inflight := MaxInflight,
-    ssl := #{enable := EnableSsl} = Ssl
-}) ->
+basic_config(
     #{
+        server := Server,
+        reconnect_interval := ReconnIntv,
+        proto_ver := ProtoVer,
+        bridge_mode := BridgeMode,
+        clean_start := CleanStart,
+        keepalive := KeepAlive,
+        retry_interval := RetryIntv,
+        max_inflight := MaxInflight,
+        ssl := #{enable := EnableSsl} = Ssl
+    } = Conf
+) ->
+    BaiscConf = #{
         %% connection opts
         server => Server,
         %% 30s
@@ -251,8 +251,6 @@ basic_config(#{
         %% non-standard mqtt connection packets will be filtered out by LB.
         %% So let's disable bridge_mode.
         bridge_mode => BridgeMode,
-        username => User,
-        password => Password,
         clean_start => CleanStart,
         keepalive => ms_to_s(KeepAlive),
         retry_interval => RetryIntv,
@@ -260,7 +258,20 @@ basic_config(#{
         ssl => EnableSsl,
         ssl_opts => maps:to_list(maps:remove(enable, Ssl)),
         if_record_metrics => true
-    }.
+    },
+    maybe_put_fields([username, password], Conf, BaiscConf).
+
+maybe_put_fields(Fields, Conf, Acc0) ->
+    lists:foldl(
+        fun(Key, Acc) ->
+            case maps:find(Key, Conf) of
+                error -> Acc;
+                {ok, Val} -> Acc#{Key => Val}
+            end
+        end,
+        Acc0,
+        Fields
+    ).
 
 ms_to_s(Ms) ->
     erlang:ceil(Ms / 1000).
diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl
index 77d371711..d15db24be 100644
--- a/apps/emqx_rule_engine/include/rule_engine.hrl
+++ b/apps/emqx_rule_engine/include/rule_engine.hrl
@@ -88,18 +88,18 @@
 %% Logical operators
 -define(is_logical(Op), (Op =:= 'and' orelse Op =:= 'or')).
 
--define(RAISE(_EXP_, _ERROR_),
-    ?RAISE(_EXP_, _ = do_nothing, _ERROR_)
+-define(RAISE(EXP, ERROR),
+    ?RAISE(EXP, _ = do_nothing, ERROR)
 ).
 
--define(RAISE(_EXP_, _EXP_ON_FAIL_, _ERROR_),
+-define(RAISE(EXP, EXP_ON_FAIL, ERROR),
     fun() ->
         try
-            (_EXP_)
+            (EXP)
         catch
-            _EXCLASS_:_EXCPTION_:_ST_ ->
-                _EXP_ON_FAIL_,
-                throw(_ERROR_)
+            EXCLASS:EXCPTION:ST ->
+                EXP_ON_FAIL,
+                throw(ERROR)
         end
     end()
 ).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index 847c4ff00..9ec69f1d7 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -130,13 +130,13 @@ do_apply_rule(
 ) ->
     {Selected, Collection} = ?RAISE(
         select_and_collect(Fields, Columns),
-        {select_and_collect_error, {_EXCLASS_, _EXCPTION_, _ST_}}
+        {select_and_collect_error, {EXCLASS, EXCPTION, ST}}
     ),
     ColumnsAndSelected = maps:merge(Columns, Selected),
     case
         ?RAISE(
             match_conditions(Conditions, ColumnsAndSelected),
-            {match_conditions_error, {_EXCLASS_, _EXCPTION_, _ST_}}
+            {match_conditions_error, {EXCLASS, EXCPTION, ST}}
         )
     of
         true ->
@@ -166,12 +166,12 @@ do_apply_rule(
 ) ->
     Selected = ?RAISE(
         select_and_transform(Fields, Columns),
-        {select_and_transform_error, {_EXCLASS_, _EXCPTION_, _ST_}}
+        {select_and_transform_error, {EXCLASS, EXCPTION, ST}}
     ),
     case
         ?RAISE(
             match_conditions(Conditions, maps:merge(Columns, Selected)),
-            {match_conditions_error, {_EXCLASS_, _EXCPTION_, _ST_}}
+            {match_conditions_error, {EXCLASS, EXCPTION, ST}}
         )
     of
         true ->
@@ -245,7 +245,7 @@ filter_collection(Columns, InCase, DoEach, {CollKey, CollVal}) ->
             case
                 ?RAISE(
                     match_conditions(InCase, ColumnsAndItem),
-                    {match_incase_error, {_EXCLASS_, _EXCPTION_, _ST_}}
+                    {match_incase_error, {EXCLASS, EXCPTION, ST}}
                 )
             of
                 true when DoEach == [] -> {true, ColumnsAndItem};
@@ -253,7 +253,7 @@ filter_collection(Columns, InCase, DoEach, {CollKey, CollVal}) ->
                     {true,
                         ?RAISE(
                             select_and_transform(DoEach, ColumnsAndItem),
-                            {doeach_error, {_EXCLASS_, _EXCPTION_, _ST_}}
+                            {doeach_error, {EXCLASS, EXCPTION, ST}}
                         )};
                 false ->
                     false

From 14633eaac876a055a0f651a30c962ee784d8d873 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 31 Aug 2022 09:40:22 +0800
Subject: [PATCH 115/232] fix: please the elvis

---
 apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index 9ec69f1d7..b1d563291 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -42,6 +42,10 @@
 -type alias() :: atom().
 -type collection() :: {alias(), [term()]}.
 
+-elvis([
+    {elvis_style, invalid_dynamic_call, #{ignore => [emqx_rule_runtime]}}
+]).
+
 -define(ephemeral_alias(TYPE, NAME),
     iolist_to_binary(io_lib:format("_v_~ts_~p_~p", [TYPE, NAME, erlang:system_time()]))
 ).
@@ -271,7 +275,7 @@ match_conditions({'not', Var}, Data) ->
     case eval(Var, Data) of
         Bool when is_boolean(Bool) ->
             not Bool;
-        _other ->
+        _Other ->
             false
     end;
 match_conditions({in, Var, {list, Vals}}, Data) ->

From ba1f5eecd32028c426b055bc4a211d0f2f5f381b Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 31 Aug 2022 11:14:36 +0800
Subject: [PATCH 116/232] fix: update the swagger for new resource metrics

---
 apps/emqx_bridge/include/emqx_bridge.hrl       | 10 ++++++++++
 .../emqx_ee_bridge/include/emqx_ee_bridge.hrl  | 18 ------------------
 .../src/emqx_ee_bridge_hstreamdb.erl           |  2 +-
 .../src/emqx_ee_bridge_influxdb.erl            |  2 +-
 .../src/emqx_ee_bridge_mysql.erl               |  2 +-
 5 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/apps/emqx_bridge/include/emqx_bridge.hrl b/apps/emqx_bridge/include/emqx_bridge.hrl
index 217c403b9..bb8ee6e29 100644
--- a/apps/emqx_bridge/include/emqx_bridge.hrl
+++ b/apps/emqx_bridge/include/emqx_bridge.hrl
@@ -87,3 +87,13 @@
         received := Rcvd
     }
 ).
+
+-define(METRICS_EXAMPLE, #{
+    metrics => ?EMPTY_METRICS,
+    node_metrics => [
+        #{
+            node => node(),
+            metrics => ?EMPTY_METRICS
+        }
+    ]
+}).
diff --git a/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl b/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl
index 0065db56b..e69de29bb 100644
--- a/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl
+++ b/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl
@@ -1,18 +0,0 @@
--define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{
-    matched => MATCH,
-    success => SUCC,
-    failed => FAILED,
-    rate => RATE,
-    rate_last5m => RATE_5,
-    rate_max => RATE_MAX
-}).
-
--define(METRICS_EXAMPLE, #{
-    metrics => ?METRICS(0, 0, 0, 0, 0, 0),
-    node_metrics => [
-        #{
-            node => node(),
-            metrics => ?METRICS(0, 0, 0, 0, 0, 0)
-        }
-    ]
-}).
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
index 3b5183150..dfae764c8 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl
@@ -5,7 +5,7 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include("emqx_ee_bridge.hrl").
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 24b4aebb1..6ad804b2c 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -3,7 +3,7 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_bridge_influxdb).
 
--include("emqx_ee_bridge.hrl").
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index 6a5d4a3a2..f76eb388e 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -5,7 +5,7 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include("emqx_ee_bridge.hrl").
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).

From 0ef0b68de48cb7e9a4127c7913401fb524417d1b Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 31 Aug 2022 18:25:00 +0800
Subject: [PATCH 117/232] refactor: change '{recoverable_error,Reason}' to
 '{error,{recoverable_error,Reason}}'

---
 .../src/emqx_connector_http.erl               | 25 ++++++++++++++-----
 .../src/emqx_connector_mysql.erl              |  2 +-
 apps/emqx_resource/include/emqx_resource.hrl  |  4 +--
 .../src/emqx_resource_worker.erl              |  8 +++---
 .../src/emqx_rule_runtime.erl                 |  2 +-
 5 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 2562bf272..3eb55c9a4 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -46,7 +46,7 @@
     namespace/0
 ]).
 
--export([check_ssl_opts/2]).
+-export([check_ssl_opts/2, validate_method/1]).
 
 -type connect_timeout() :: emqx_schema:duration() | infinity.
 -type pool_type() :: random | hash.
@@ -137,8 +137,10 @@ fields(config) ->
 fields("request") ->
     [
         {method,
-            hoconsc:mk(hoconsc:enum([post, put, get, delete]), #{
-                required => false, desc => ?DESC("method")
+            hoconsc:mk(binary(), #{
+                required => false,
+                desc => ?DESC("method"),
+                validator => fun ?MODULE:validate_method/1
             })},
         {path, hoconsc:mk(binary(), #{required => false, desc => ?DESC("path")})},
         {body, hoconsc:mk(binary(), #{required => false, desc => ?DESC("body")})},
@@ -171,6 +173,17 @@ desc(_) ->
 validations() ->
     [{check_ssl_opts, fun check_ssl_opts/1}].
 
+validate_method(M) when M =:= <<"post">>; M =:= <<"put">>; M =:= <<"get">>; M =:= <<"delete">> ->
+    ok;
+validate_method(M) ->
+    case string:find(M, "${") of
+        nomatch ->
+            {error,
+                <<"Invalid method, should be one of 'post', 'put', 'get', 'delete' or variables in ${field} format.">>};
+        _ ->
+            ok
+    end.
+
 sc(Type, Meta) -> hoconsc:mk(Type, Meta).
 ref(Field) -> hoconsc:ref(?MODULE, Field).
 
@@ -286,13 +299,13 @@ on_query(
             Retry
         )
     of
-        {error, econnrefused} ->
+        {error, Reason} when Reason =:= econnrefused; Reason =:= timeout ->
             ?SLOG(warning, #{
                 msg => "http_connector_do_request_failed",
-                reason => econnrefused,
+                reason => Reason,
                 connector => InstId
             }),
-            {recoverable_error, econnrefused};
+            {error, {recoverable_error, Reason}};
         {error, Reason} = Result ->
             ?SLOG(error, #{
                 msg => "http_connector_do_request_failed",
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index b9c200316..802427134 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -420,7 +420,7 @@ on_sql_query(
                 error,
                 LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
             ),
-            {recoverable_error, Reason};
+            {error, {recoverable_error, Reason}};
         {error, Reason} ->
             ?SLOG(
                 error,
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 2409a7069..aab0129d1 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -76,8 +76,8 @@
 -type query_result() ::
     ok
     | {ok, term()}
-    | {error, term()}
-    | {recoverable_error, term()}.
+    | {error, {recoverable_error, term()}}
+    | {error, term()}.
 
 -define(WORKER_POOL_SIZE, 16).
 
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 3a12a6b91..d951d7a9f 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -396,7 +396,7 @@ handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
 handle_query_result(Id, {error, _}, BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
-handle_query_result(Id, {recoverable_error, _}, _BlockWorker) ->
+handle_query_result(Id, {error, {recoverable_error, _}}, _BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', -1),
     true;
 handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
@@ -433,7 +433,7 @@ call_query(QM0, Id, Query, QueryOpts) ->
     try
         %% if the callback module (connector) wants to return an error that
         %% makes the current resource goes into the `blocked` state, it should
-        %% return `{recoverable_error, Reason}`
+        %% return `{error, {recoverable_error, Reason}}`
         EXPR
     catch
         ERR:REASON:STACKTRACE ->
@@ -457,7 +457,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?APPLY_RESOURCE(
         case inflight_is_full(Name, WinSize) of
             true ->
-                ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
+                ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
                 ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
@@ -483,7 +483,7 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?APPLY_RESOURCE(
         case inflight_is_full(Name, WinSize) of
             true ->
-                ?tp(inflight_full, #{id => Id, wind_size => WinSize}),
+                ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
                 ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', length(Batch)),
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index b1d563291..29f2c6bf6 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -510,7 +510,7 @@ nested_put(Alias, Val, Columns0) ->
 -define(IS_RES_DOWN(R), R == stopped; R == not_connected; R == not_found).
 inc_action_metrics(ok, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success');
-inc_action_metrics({recoverable_error, _}, RuleId) ->
+inc_action_metrics({error, {recoverable_error, _}}, RuleId) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');
 inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) ->
     emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');

From dac178cbaf5a80c857c3b22e85363694963679d4 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 31 Aug 2022 17:23:47 +0200
Subject: [PATCH 118/232] chore: ensure version bumps

---
 apps/emqx_authz/src/emqx_authz.app.src         |  2 +-
 apps/emqx_bridge/src/emqx_bridge.app.src       |  2 +-
 apps/emqx_dashboard/src/emqx_dashboard.app.src |  2 +-
 scripts/apps-version-check.sh                  | 12 ++++++++++--
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index 940d73d0d..a0ce3e170 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src
index 3be8d38fd..8e1c60c29 100644
--- a/apps/emqx_bridge/src/emqx_bridge.app.src
+++ b/apps/emqx_bridge/src/emqx_bridge.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "An OTP application"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index 4e1a3518f..adf974243 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.4"},
+    {vsn, "5.0.5"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},
diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh
index 72ddd5dc3..ef378549f 100755
--- a/scripts/apps-version-check.sh
+++ b/scripts/apps-version-check.sh
@@ -19,9 +19,17 @@ while read -r app; do
         app_path="."
     fi
     src_file="$app_path/src/$(basename "$app").app.src"
-    old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')"
+    if git show "$latest_release":"$src_file" >/dev/null 2>&1; then
+        old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')"
+    else
+        old_app_version='not_found'
+    fi
     now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
-    if [ "$old_app_version" = "$now_app_version" ]; then
+
+    if [ "$old_app_version" = 'not_found' ]; then
+        echo "$src_file is newly added"
+        true
+    elif [ "$old_app_version" = "$now_app_version" ]; then
         changed_lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \
                              -- "$app_path/src" \
                              -- "$app_path/include" \

From c1afb34a865e35b48b821dc692f10b7295d7f1e2 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 1 Sep 2022 11:28:38 +0800
Subject: [PATCH 119/232] test: fix failed tests

---
 apps/emqx/test/emqx_mqtt_SUITE.erl             |  4 ++--
 apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 10 ++++++++--
 apps/emqx_bridge/src/emqx_bridge_api.erl       |  7 +++----
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/apps/emqx/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl
index f43804991..7032e553c 100644
--- a/apps/emqx/test/emqx_mqtt_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_SUITE.erl
@@ -115,7 +115,7 @@ message_expiry_interval_init() ->
 message_expiry_interval_exipred(CPublish, CControl, QoS) ->
     ct:pal("~p ~p", [?FUNCTION_NAME, QoS]),
     %% publish to t/a and waiting for the message expired
-    emqtt:publish(
+    _ = emqtt:publish(
         CPublish,
         <<"t/a">>,
         #{'Message-Expiry-Interval' => 1},
@@ -152,7 +152,7 @@ message_expiry_interval_exipred(CPublish, CControl, QoS) ->
 message_expiry_interval_not_exipred(CPublish, CControl, QoS) ->
     ct:pal("~p ~p", [?FUNCTION_NAME, QoS]),
     %% publish to t/a
-    emqtt:publish(
+    _ = emqtt:publish(
         CPublish,
         <<"t/a">>,
         #{'Message-Expiry-Interval' => 20},
diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
index 88ccda452..cb71cef95 100644
--- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
@@ -529,8 +529,11 @@ t_connack_max_qos_allowed(Config) ->
     %% [MQTT-3.2.2-10]
     {ok, _, [2]} = emqtt:subscribe(Client1, Topic, 2),
 
-    {ok, _} = emqtt:publish(Client1, Topic, <<"Unsupported Qos 1">>, qos1),
     %% [MQTT-3.2.2-11]
+    ?assertMatch(
+        {error, {disconnected, 155, _}},
+        emqtt:publish(Client1, Topic, <<"Unsupported Qos 1">>, qos1)
+    ),
     ?assertEqual(155, receive_disconnect_reasoncode()),
     waiting_client_process_exit(Client1),
 
@@ -563,8 +566,11 @@ t_connack_max_qos_allowed(Config) ->
     %% [MQTT-3.2.2-10]
     {ok, _, [2]} = emqtt:subscribe(Client3, Topic, 2),
 
-    {ok, _} = emqtt:publish(Client3, Topic, <<"Unsupported Qos 2">>, qos2),
     %% [MQTT-3.2.2-11]
+    ?assertMatch(
+        {error, {disconnected, 155, _}},
+        emqtt:publish(Client3, Topic, <<"Unsupported Qos 2">>, qos2)
+    ),
     ?assertEqual(155, receive_disconnect_reasoncode()),
     waiting_client_process_exit(Client3),
 
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index a8bc207d6..bd892edcd 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -20,7 +20,6 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
--include_lib("emqx_resource/include/emqx_resource.hrl").
 -include_lib("emqx_bridge/include/emqx_bridge.hrl").
 
 -import(hoconsc, [mk/2, array/1, enum/1]).
@@ -232,11 +231,11 @@ mqtt_main_example() ->
         retry_interval => <<"15s">>,
         max_inflight => 100,
         resource_opts => #{
-            health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
-            auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
+            health_check_interval => <<"15s">>,
+            auto_restart_interval => <<"60s">>,
             query_mode => sync,
             enable_queue => false,
-            max_queue_bytes => ?DEFAULT_QUEUE_SIZE
+            max_queue_bytes => 100 * 1024 * 1024
         },
         ssl => #{
             enable => false

From dda0b4ac8cb98c1a9c5ecd4165bfa3a41548cefd Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 1 Sep 2022 12:35:37 +0800
Subject: [PATCH 120/232] chore: update emqtt vsn

---
 apps/emqx/rebar.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 5b5b3db39..8e0d1f71c 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -43,7 +43,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.6.0"}}}
+            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0-rc.1"}}}
         ]},
         {extra_src_dirs, [{"test", [recursive]}]}
     ]}

From 33c9c7d4979e0dfef2adeb00a6e81f61a6153530 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 1 Sep 2022 14:51:13 +0800
Subject: [PATCH 121/232] fix: incorrect message order when batch is enabled

---
 .../src/emqx_resource_worker.erl              | 29 +++++++++++++++----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index d951d7a9f..a0632a761 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -303,11 +303,12 @@ flush(#{acc := []} = St) ->
 flush(
     #{
         id := Id,
-        acc := Batch,
+        acc := Batch0,
         batch_size := Size,
         queue := Q0
     } = St
 ) ->
+    Batch = lists:reverse(Batch0),
     QueryOpts = #{
         inflight_name => maps:get(name, St),
         inflight_window => maps:get(async_inflight_window, St)
@@ -393,12 +394,12 @@ handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.other'),
     BlockWorker;
-handle_query_result(Id, {error, _}, BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
-    BlockWorker;
 handle_query_result(Id, {error, {recoverable_error, _}}, _BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', -1),
     true;
+handle_query_result(Id, {error, _}, BlockWorker) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
+    BlockWorker;
 handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
     true;
 handle_query_result(Id, {async_return, {error, _}}, BlockWorker) ->
@@ -449,7 +450,15 @@ call_query(QM0, Id, Query, QueryOpts) ->
 apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
-    ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request);
+    ?APPLY_RESOURCE(
+        begin
+            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
+            Result = Mod:on_query(Id, Request, ResSt),
+            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
+            Result
+        end,
+        Request
+    );
 apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
@@ -475,7 +484,15 @@ apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
     ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', length(Batch)),
-    ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch);
+    ?APPLY_RESOURCE(
+        begin
+            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
+            Result = Mod:on_batch_query(Id, Requests, ResSt),
+            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
+            Result
+        end,
+        Batch
+    );
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),

From 3d4afd65dfcb545a0e453864bc924d2962917de5 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 30 Aug 2022 15:47:29 -0300
Subject: [PATCH 122/232] feat: add mongodb bridge (e5.0)

---
 .../docker-compose-mongo-replicaset-tcp.yaml  |   8 +-
 .../docker-compose-mongo-sharded-tcp.yaml     |  90 +++++++
 .github/workflows/elixir_release.yml          |  19 +-
 .github/workflows/run_test_cases.yaml         |   7 +
 .../src/schema/emqx_bridge_schema.erl         |  18 ++
 .../src/emqx_connector_mongo.erl              |  45 +++-
 .../i18n/emqx_ee_bridge_mongodb.conf          |  67 +++++
 lib-ee/emqx_ee_bridge/rebar.config            |   6 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  18 +-
 .../src/emqx_ee_bridge_mongodb.erl            | 154 +++++++++++
 .../test/emqx_ee_bridge_mongodb_SUITE.erl     | 251 ++++++++++++++++++
 lib-ee/emqx_ee_connector/rebar.config         |   3 +-
 scripts/docker-ct-apps                        |   1 +
 13 files changed, 673 insertions(+), 14 deletions(-)
 create mode 100644 .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml
 create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl
 create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl

diff --git a/.ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml b/.ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml
index f83fe0932..54506abd8 100644
--- a/.ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml
+++ b/.ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml
@@ -18,7 +18,7 @@ services:
       --ipv6
       --bind_ip_all
       --replSet rs0
-      
+
   mongo2:
     hostname: mongo2
     container_name: mongo2
@@ -54,10 +54,10 @@ services:
       --ipv6
       --bind_ip_all
       --replSet rs0
-      
-  mongo_client:
+
+  mongo_rs_client:
     image: mongo:${MONGO_TAG}
-    container_name: mongo_client
+    container_name: mongo_rs_client
     networks:
       - emqx_bridge
     depends_on:
diff --git a/.ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml b/.ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml
new file mode 100644
index 000000000..a8b51689b
--- /dev/null
+++ b/.ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml
@@ -0,0 +1,90 @@
+version: "3"
+
+services:
+  mongosharded1:
+    hostname: mongosharded1
+    container_name: mongosharded1
+    image: mongo:${MONGO_TAG}
+    environment:
+      MONGO_INITDB_DATABASE: mqtt
+    networks:
+      - emqx_bridge
+    expose:
+      - 27017
+    ports:
+      - 27014:27017
+    restart: always
+    command:
+      --configsvr
+      --replSet cfg0
+      --port 27017
+      --ipv6
+      --bind_ip_all
+
+  mongosharded2:
+    hostname: mongosharded2
+    container_name: mongosharded2
+    image: mongo:${MONGO_TAG}
+    environment:
+      MONGO_INITDB_DATABASE: mqtt
+    networks:
+      - emqx_bridge
+    expose:
+      - 27017
+    ports:
+      - 27015:27017
+    restart: always
+    command:
+      --shardsvr
+      --replSet rs0
+      --port 27017
+      --ipv6
+      --bind_ip_all
+
+  mongosharded3:
+    hostname: mongosharded3
+    container_name: mongosharded3
+    image: mongo:${MONGO_TAG}
+    environment:
+      MONGO_INITDB_DATABASE: mqtt
+    networks:
+      - emqx_bridge
+    expose:
+      - 27017
+    ports:
+      - 27016:27017
+    restart: always
+    entrypoint: mongos
+    command:
+      --configdb cfg0/mongosharded1:27017
+      --port 27017
+      --ipv6
+      --bind_ip_all
+
+  mongosharded_client:
+    image: mongo:${MONGO_TAG}
+    container_name: mongosharded_client
+    networks:
+      - emqx_bridge
+    depends_on:
+      - mongosharded1
+      - mongosharded2
+      - mongosharded3
+    command:
+      - /bin/bash
+      - -c
+      - |
+        while ! mongo --host mongosharded1 --eval 'db.runCommand("ping").ok' --quiet >/dev/null 2>&1 ; do
+            sleep 1
+        done
+        mongo --host mongosharded1 --eval "rs.initiate( { _id : 'cfg0', configsvr: true, members: [ { _id : 0, host : 'mongosharded1:27017' } ] })"
+        while ! mongo --host mongosharded2 --eval 'db.runCommand("ping").ok' --quiet >/dev/null 2>&1  ; do
+            sleep 1
+        done
+        mongo --host mongosharded2 --eval "rs.initiate( { _id : 'rs0', members: [ { _id : 0, host : 'mongosharded2:27017' } ] })"
+        mongo --host mongosharded2 --eval "rs.status()"
+        while ! mongo --host mongosharded3 --eval 'db.runCommand("ping").ok' --quiet >/dev/null 2>&1  ; do
+            sleep 1
+        done
+        mongo --host mongosharded3 --eval "sh.addShard('rs0/mongosharded2:27017')"
+        mongo --host mongosharded3 --eval "sh.enableSharding('mqtt')"
diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml
index 006d6aba8..b703f2fde 100644
--- a/.github/workflows/elixir_release.yml
+++ b/.github/workflows/elixir_release.yml
@@ -12,7 +12,18 @@ on:
 jobs:
   elixir_release_build:
     runs-on: ubuntu-latest
-    container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
+    strategy:
+      matrix:
+        otp:
+          - 24.2.1-1
+        elixir:
+          - 1.13.4
+        os:
+          - ubuntu20.04
+        profile:
+          - emqx
+          - emqx-enterprise
+    container: ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
 
     steps:
       - name: Checkout
@@ -23,15 +34,15 @@ jobs:
         run: |
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: elixir release
-        run: make emqx-elixir
+        run: make ${{ matrix.profile }}-elixir
       - name: start release
         run: |
-          cd _build/emqx/rel/emqx
+          cd _build/${{ matrix.profile }}/rel/emqx
           bin/emqx start
       - name: check if started
         run: |
           sleep 10
           nc -zv localhost 1883
-          cd _build/emqx/rel/emqx
+          cd _build/${{ matrix.profile }}/rel/emqx
           bin/emqx ping
           bin/emqx ctl status
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index b4b797621..5d0dd1ca2 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -124,6 +124,8 @@ jobs:
             docker-compose \
                 -f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
                 -f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \
+                -f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \
+                -f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \
                 -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
                 -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \
                 -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
@@ -135,6 +137,11 @@ jobs:
                 -f .ci/docker-compose-file/docker-compose.yaml \
                 up -d --build
 
+        - name: wait some services to be fully up
+          run: |
+            docker wait mongosharded_client
+            docker wait mongo_rs_client
+
           # produces <app-name>.coverdata
         - name: run common test
           working-directory: source
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index f55ac840e..af74ae2ec 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -41,12 +41,16 @@ api_schema(Method) ->
     hoconsc:union(Broker ++ EE).
 
 ee_api_schemas(Method) ->
+    %% must ensure the app is loaded before checking if fn is defined.
+    ensure_loaded(emqx_ee_bridge, emqx_ee_bridge),
     case erlang:function_exported(emqx_ee_bridge, api_schemas, 1) of
         true -> emqx_ee_bridge:api_schemas(Method);
         false -> []
     end.
 
 ee_fields_bridges() ->
+    %% must ensure the app is loaded before checking if fn is defined.
+    ensure_loaded(emqx_ee_bridge, emqx_ee_bridge),
     case erlang:function_exported(emqx_ee_bridge, fields, 1) of
         true -> emqx_ee_bridge:fields(bridges);
         false -> []
@@ -155,3 +159,17 @@ status() ->
 
 node_name() ->
     {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}.
+
+%%=================================================================================================
+%% Internal fns
+%%=================================================================================================
+
+ensure_loaded(App, Mod) ->
+    try
+        _ = application:load(App),
+        _ = Mod:module_info(),
+        ok
+    catch
+        _:_ ->
+            ok
+    end.
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index 07208545f..5bfcbe009 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -37,7 +37,7 @@
 
 -export([roots/0, fields/1, desc/1]).
 
--export([mongo_query/5, check_worker_health/1]).
+-export([mongo_query/5, mongo_insert/3, check_worker_health/1]).
 
 -define(HEALTH_CHECK_TIMEOUT, 30000).
 
@@ -177,9 +177,16 @@ on_start(
         {worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}
     ],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
+    Collection = maps:get(collection, Config, <<"mqtt">>),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts) of
-        ok -> {ok, #{poolname => PoolName, type => Type}};
-        {error, Reason} -> {error, Reason}
+        ok ->
+            {ok, #{
+                poolname => PoolName,
+                type => Type,
+                collection => Collection
+            }};
+        {error, Reason} ->
+            {error, Reason}
     end.
 
 on_stop(InstId, #{poolname := PoolName}) ->
@@ -189,6 +196,35 @@ on_stop(InstId, #{poolname := PoolName}) ->
     }),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
+on_query(
+    InstId,
+    {send_message, Document},
+    #{poolname := PoolName, collection := Collection} = State
+) ->
+    Request = {insert, Collection, Document},
+    ?TRACE(
+        "QUERY",
+        "mongodb_connector_received",
+        #{request => Request, connector => InstId, state => State}
+    ),
+    case
+        ecpool:pick_and_do(
+            PoolName,
+            {?MODULE, mongo_insert, [Collection, Document]},
+            no_handover
+        )
+    of
+        {{false, Reason}, _Document} ->
+            ?SLOG(error, #{
+                msg => "mongodb_connector_do_query_failed",
+                request => Request,
+                reason => Reason,
+                connector => InstId
+            }),
+            {error, Reason};
+        {{true, _Info}, _Document} ->
+            ok
+    end;
 on_query(
     InstId,
     {Action, Collection, Filter, Projector},
@@ -292,6 +328,9 @@ mongo_query(Conn, find_one, Collection, Filter, Projector) ->
 mongo_query(_Conn, _Action, _Collection, _Filter, _Projector) ->
     ok.
 
+mongo_insert(Conn, Collection, Documents) ->
+    mongo_api:insert(Conn, Collection, Documents).
+
 init_type(#{mongo_type := rs, replica_set_name := ReplicaSetName}) ->
     {rs, ReplicaSetName};
 init_type(#{mongo_type := Type}) ->
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf
new file mode 100644
index 000000000..fef3663ef
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf
@@ -0,0 +1,67 @@
+emqx_ee_bridge_mongodb {
+  desc_config {
+    desc {
+      en: "Configuration for MongoDB Bridge"
+      zh: "为MongoDB桥配置"
+    }
+    label {
+      en: "MongoDB Bridge Configuration"
+      zh: "MongoDB桥配置"
+    }
+  }
+
+  enable {
+    desc {
+      en: "Enable or disable this MongoDB Bridge"
+      zh: "启用或停用该MongoDB桥"
+    }
+    label {
+      en: "Enable or disable"
+      zh: "启用或禁用"
+    }
+  }
+
+  collection {
+    desc {
+      en: "The collection where data will be stored into"
+      zh: "数据将被存储到的集合"
+    }
+    label {
+      en: "Collection to be used"
+      zh: "将要使用的藏品"
+    }
+  }
+
+  mongodb_rs_conf {
+    desc {
+      en: "MongoDB (Replica Set) configuration"
+      zh: "MongoDB（Replica Set）配置"
+    }
+    label {
+      en: "MongoDB (Replica Set) Configuration"
+      zh: "MongoDB（Replica Set）配置"
+    }
+  }
+
+  mongodb_sharded_conf {
+    desc {
+      en: "MongoDB (Sharded) configuration"
+      zh: "MongoDB (Sharded)配置"
+    }
+    label {
+      en: "MongoDB (Sharded) Configuration"
+      zh: "MongoDB (Sharded)配置"
+    }
+  }
+
+  mongodb_single_conf {
+    desc {
+      en: "MongoDB (Standalone) configuration"
+      zh: "MongoDB（独立）配置"
+    }
+    label {
+      en: "MongoDB (Standalone) Configuration"
+      zh: "MongoDB（独立）配置"
+    }
+  }
+}
diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config
index 5dd22ccef..e986d7983 100644
--- a/lib-ee/emqx_ee_bridge/rebar.config
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@@ -1,5 +1,9 @@
 {erl_opts, [debug_info]}.
-{deps, []}.
+{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}
+       , {emqx_connector, {path, "../../apps/emqx_connector"}}
+       , {emqx_resource, {path, "../../apps/emqx_resource"}}
+       , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
+       ]}.
 
 {shell, [
     {apps, [emqx_ee_bridge]}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index ad1dfaee8..840b963cd 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -15,6 +15,9 @@
 api_schemas(Method) ->
     [
         ref(emqx_ee_bridge_mysql, Method),
+        ref(emqx_ee_bridge_mongodb, Method ++ "_rs"),
+        ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"),
+        ref(emqx_ee_bridge_mongodb, Method ++ "_single"),
         ref(emqx_ee_bridge_hstreamdb, Method),
         ref(emqx_ee_bridge_influxdb, Method ++ "_udp"),
         ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"),
@@ -25,6 +28,7 @@ schema_modules() ->
     [
         emqx_ee_bridge_hstreamdb,
         emqx_ee_bridge_influxdb,
+        emqx_ee_bridge_mongodb,
         emqx_ee_bridge_mysql
     ].
 
@@ -42,6 +46,9 @@ examples(Method) ->
 
 resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8));
 resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb;
+resource_type(mongodb_rs) -> emqx_connector_mongo;
+resource_type(mongodb_sharded) -> emqx_connector_mongo;
+resource_type(mongodb_single) -> emqx_connector_mongo;
 resource_type(mysql) -> emqx_connector_mysql;
 resource_type(influxdb_udp) -> emqx_ee_connector_influxdb;
 resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb;
@@ -59,7 +66,16 @@ fields(bridges) ->
                 hoconsc:map(name, ref(emqx_ee_bridge_mysql, "config")),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
-    ] ++ fields(influxdb);
+    ] ++ fields(mongodb) ++ fields(influxdb);
+fields(mongodb) ->
+    [
+        {Type,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_bridge_mongodb, Type)),
+                #{desc => <<"EMQX Enterprise Config">>}
+            )}
+     || Type <- [mongodb_rs, mongodb_sharded, mongodb_single]
+    ];
 fields(influxdb) ->
     [
         {Protocol,
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl
new file mode 100644
index 000000000..9d9a5e4d0
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl
@@ -0,0 +1,154 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_mongodb).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-behaviour(hocon_schema).
+
+%% emqx_ee_bridge "callbacks"
+-export([
+    conn_bridge_examples/1
+]).
+
+%% hocon_schema callbacks
+-export([
+    namespace/0,
+    roots/0,
+    fields/1,
+    desc/1
+]).
+
+%%=================================================================================================
+%% hocon_schema API
+%%=================================================================================================
+
+namespace() ->
+    "bridge_mongodb".
+
+roots() ->
+    [].
+
+fields("config") ->
+    [
+        {enable, mk(boolean(), #{desc => ?DESC("enable"), default => true})},
+        {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}
+    ];
+fields(mongodb_rs) ->
+    emqx_connector_mongo:fields(rs) ++ fields("config");
+fields(mongodb_sharded) ->
+    emqx_connector_mongo:fields(sharded) ++ fields("config");
+fields(mongodb_single) ->
+    emqx_connector_mongo:fields(single) ++ fields("config");
+fields("post_rs") ->
+    fields(mongodb_rs);
+fields("post_sharded") ->
+    fields(mongodb_sharded);
+fields("post_single") ->
+    fields(mongodb_single);
+fields("put_rs") ->
+    fields(mongodb_rs);
+fields("put_sharded") ->
+    fields(mongodb_sharded);
+fields("put_single") ->
+    fields(mongodb_single);
+fields("get_rs") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields(mongodb_rs);
+fields("get_sharded") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields(mongodb_sharded);
+fields("get_single") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields(mongodb_single).
+
+conn_bridge_examples(Method) ->
+    [
+        #{
+            <<"mongodb_rs">> => #{
+                summary => <<"MongoDB (Replica Set) Bridge">>,
+                value => values(mongodb_rs, Method)
+            }
+        },
+        #{
+            <<"mongodb_sharded">> => #{
+                summary => <<"MongoDB (Sharded) Bridge">>,
+                value => values(mongodb_sharded, Method)
+            }
+        },
+        #{
+            <<"mongodb_single">> => #{
+                summary => <<"MongoDB (Standalone) Bridge">>,
+                value => values(mongodb_single, Method)
+            }
+        }
+    ].
+
+desc("config") ->
+    ?DESC("desc_config");
+desc(mongodb_rs) ->
+    ?DESC(mongodb_rs_conf);
+desc(mongodb_sharded) ->
+    ?DESC(mongodb_sharded_conf);
+desc(mongodb_single) ->
+    ?DESC(mongodb_single_conf);
+desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
+    ["Configuration for MongoDB using `", string:to_upper(Method), "` method."];
+desc(_) ->
+    undefined.
+
+%%=================================================================================================
+%% Internal fns
+%%=================================================================================================
+
+values(mongodb_rs = MongoType, Method) ->
+    TypeOpts = #{
+        servers => <<"localhost:27017, localhost:27018">>,
+        w_mode => <<"safe">>,
+        r_mode => <<"safe">>,
+        replica_set_name => <<"rs">>
+    },
+    values(common, MongoType, Method, TypeOpts);
+values(mongodb_sharded = MongoType, Method) ->
+    TypeOpts = #{
+        servers => <<"localhost:27017, localhost:27018">>,
+        w_mode => <<"safe">>
+    },
+    values(common, MongoType, Method, TypeOpts);
+values(mongodb_single = MongoType, Method) ->
+    TypeOpts = #{
+        server => <<"localhost:27017">>,
+        w_mode => <<"safe">>
+    },
+    values(common, MongoType, Method, TypeOpts).
+
+values(common, MongoType, Method, TypeOpts) ->
+    MongoTypeBin = atom_to_binary(MongoType),
+    Common = #{
+        name => <<MongoTypeBin/binary, "_demo">>,
+        type => MongoTypeBin,
+        enable => true,
+        collection => <<"mycol">>,
+        database => <<"mqtt">>,
+        srv_record => false,
+        pool_size => 8,
+        username => <<"myuser">>,
+        password => <<"mypass">>
+    },
+    MethodVals = method_values(MongoType, Method),
+    Vals0 = maps:merge(MethodVals, Common),
+    maps:merge(Vals0, TypeOpts).
+
+method_values(MongoType, get) ->
+    Vals = method_values(MongoType, post),
+    maps:merge(?METRICS_EXAMPLE, Vals);
+method_values(MongoType, _) ->
+    ConnectorType =
+        case MongoType of
+            mongodb_rs -> <<"rs">>;
+            mongodb_sharded -> <<"sharded">>;
+            mongodb_single -> <<"single">>
+        end,
+    #{mongo_type => ConnectorType}.
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
new file mode 100644
index 000000000..775acf7b0
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
@@ -0,0 +1,251 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_bridge_mongodb_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+%%------------------------------------------------------------------------------
+%% CT boilerplate
+%%------------------------------------------------------------------------------
+
+all() ->
+    [
+        {group, rs},
+        {group, sharded},
+        {group, single}
+        | (emqx_common_test_helpers:all(?MODULE) -- group_tests())
+    ].
+
+group_tests() ->
+    [
+        t_setup_via_config_and_publish,
+        t_setup_via_http_api_and_publish
+    ].
+
+groups() ->
+    [
+        {rs, group_tests()},
+        {sharded, group_tests()},
+        {single, group_tests()}
+    ].
+
+init_per_group(Type = rs, Config) ->
+    MongoHost = os:getenv("MONGO_RS_HOST", "mongo1"),
+    MongoPort = list_to_integer(os:getenv("MONGO_RS_PORT", "27017")),
+    case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
+        true ->
+            _ = application:load(emqx_ee_bridge),
+            _ = emqx_ee_bridge:module_info(),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            emqx_mgmt_api_test_util:init_suite(),
+            MongoConfig = mongo_config(MongoHost, MongoPort, Type),
+            [
+                {mongo_host, MongoHost},
+                {mongo_port, MongoPort},
+                {mongo_config, MongoConfig}
+                | Config
+            ];
+        false ->
+            {skip, no_mongo}
+    end;
+init_per_group(Type = sharded, Config) ->
+    MongoHost = os:getenv("MONGO_SHARDED_HOST", "mongosharded3"),
+    MongoPort = list_to_integer(os:getenv("MONGO_SHARDED_PORT", "27017")),
+    case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
+        true ->
+            _ = application:load(emqx_ee_bridge),
+            _ = emqx_ee_bridge:module_info(),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            emqx_mgmt_api_test_util:init_suite(),
+            MongoConfig = mongo_config(MongoHost, MongoPort, Type),
+            [
+                {mongo_host, MongoHost},
+                {mongo_port, MongoPort},
+                {mongo_config, MongoConfig}
+                | Config
+            ];
+        false ->
+            {skip, no_mongo}
+    end;
+init_per_group(Type = single, Config) ->
+    MongoHost = os:getenv("MONGO_SINGLE_HOST", "mongo"),
+    MongoPort = list_to_integer(os:getenv("MONGO_SINGLE_PORT", "27017")),
+    case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
+        true ->
+            _ = application:load(emqx_ee_bridge),
+            _ = emqx_ee_bridge:module_info(),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            emqx_mgmt_api_test_util:init_suite(),
+            MongoConfig = mongo_config(MongoHost, MongoPort, Type),
+            [
+                {mongo_host, MongoHost},
+                {mongo_port, MongoPort},
+                {mongo_config, MongoConfig}
+                | Config
+            ];
+        false ->
+            {skip, no_mongo}
+    end.
+
+end_per_group(_Type, _Config) ->
+    ok.
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_mgmt_api_test_util:end_suite(),
+    ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
+    ok.
+
+init_per_testcase(_Testcase, Config) ->
+    catch clear_db(Config),
+    delete_bridge(Config),
+    Config.
+
+end_per_testcase(_Testcase, Config) ->
+    catch clear_db(Config),
+    delete_bridge(Config),
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Helper fns
+%%------------------------------------------------------------------------------
+
+mongo_config(MongoHost0, MongoPort0, rs) ->
+    MongoHost = list_to_binary(MongoHost0),
+    MongoPort = integer_to_binary(MongoPort0),
+    Servers = <<MongoHost/binary, ":", MongoPort/binary>>,
+    Name = atom_to_binary(?MODULE),
+    #{
+        <<"type">> => <<"mongodb_rs">>,
+        <<"name">> => Name,
+        <<"enable">> => true,
+        <<"collection">> => <<"mycol">>,
+        <<"servers">> => Servers,
+        <<"database">> => <<"mqtt">>,
+        <<"w_mode">> => <<"safe">>,
+        <<"replica_set_name">> => <<"rs0">>
+    };
+mongo_config(MongoHost0, MongoPort0, sharded) ->
+    MongoHost = list_to_binary(MongoHost0),
+    MongoPort = integer_to_binary(MongoPort0),
+    Servers = <<MongoHost/binary, ":", MongoPort/binary>>,
+    Name = atom_to_binary(?MODULE),
+    #{
+        <<"type">> => <<"mongodb_sharded">>,
+        <<"name">> => Name,
+        <<"enable">> => true,
+        <<"collection">> => <<"mycol">>,
+        <<"servers">> => Servers,
+        <<"database">> => <<"mqtt">>,
+        <<"w_mode">> => <<"safe">>
+    };
+mongo_config(MongoHost0, MongoPort0, single) ->
+    MongoHost = list_to_binary(MongoHost0),
+    MongoPort = integer_to_binary(MongoPort0),
+    Server = <<MongoHost/binary, ":", MongoPort/binary>>,
+    Name = atom_to_binary(?MODULE),
+    #{
+        <<"type">> => <<"mongodb_single">>,
+        <<"name">> => Name,
+        <<"enable">> => true,
+        <<"collection">> => <<"mycol">>,
+        <<"server">> => Server,
+        <<"database">> => <<"mqtt">>,
+        <<"w_mode">> => <<"safe">>
+    }.
+
+create_bridge(Config0 = #{<<"type">> := Type, <<"name">> := Name}) ->
+    Config = maps:without(
+        [
+            <<"type">>,
+            <<"name">>
+        ],
+        Config0
+    ),
+    emqx_bridge:create(Type, Name, Config).
+
+delete_bridge(Config) ->
+    #{
+        <<"type">> := Type,
+        <<"name">> := Name
+    } = ?config(mongo_config, Config),
+    emqx_bridge:remove(Type, Name).
+
+create_bridge_http(Params) ->
+    Path = emqx_mgmt_api_test_util:api_path(["bridges"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of
+        {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
+        Error -> Error
+    end.
+
+clear_db(Config) ->
+    #{
+        <<"name">> := Name,
+        <<"type">> := Type,
+        <<"collection">> := Collection
+    } = ?config(mongo_config, Config),
+    ResourceID = emqx_bridge_resource:resource_id(Type, Name),
+    {ok, _, #{state := #{poolname := PoolName}}} = emqx_resource:get_instance(ResourceID),
+    Selector = #{},
+    {true, _} = ecpool:pick_and_do(
+        PoolName, {mongo_api, delete, [Collection, Selector]}, no_handover
+    ),
+    ok.
+
+find_all(Config) ->
+    #{
+        <<"name">> := Name,
+        <<"type">> := Type,
+        <<"collection">> := Collection
+    } = ?config(mongo_config, Config),
+    ResourceID = emqx_bridge_resource:resource_id(Type, Name),
+    emqx_resource:query(ResourceID, {find, Collection, #{}, #{}}).
+
+send_message(Config, Payload) ->
+    #{
+        <<"name">> := Name,
+        <<"type">> := Type
+    } = ?config(mongo_config, Config),
+    BridgeID = emqx_bridge_resource:bridge_id(Type, Name),
+    emqx_bridge:send_message(BridgeID, Payload).
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_setup_via_config_and_publish(Config) ->
+    MongoConfig = ?config(mongo_config, Config),
+    ?assertMatch(
+        {ok, _},
+        create_bridge(MongoConfig)
+    ),
+    Val = erlang:unique_integer(),
+    ok = send_message(Config, #{key => Val}),
+    ?assertMatch(
+        {ok, [#{<<"key">> := Val}]},
+        find_all(Config)
+    ),
+    ok.
+
+t_setup_via_http_api_and_publish(Config) ->
+    MongoConfig = ?config(mongo_config, Config),
+    ?assertMatch(
+        {ok, _},
+        create_bridge_http(MongoConfig)
+    ),
+    Val = erlang:unique_integer(),
+    ok = send_message(Config, #{key => Val}),
+    ?assertMatch(
+        {ok, [#{<<"key">> := Val}]},
+        find_all(Config)
+    ),
+    ok.
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index 5963b7ab0..1419c2070 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -1,7 +1,8 @@
 {erl_opts, [debug_info]}.
 {deps, [
   {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
-  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.4"}}}
+  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.4"}}},
+  {emqx, {path, "../../apps/emqx"}}
 ]}.
 
 {shell, [
diff --git a/scripts/docker-ct-apps b/scripts/docker-ct-apps
index 882a68285..e00f8dd1f 100644
--- a/scripts/docker-ct-apps
+++ b/scripts/docker-ct-apps
@@ -2,3 +2,4 @@
 apps/emqx_authn
 apps/emqx_authz
 apps/emqx_connector
+lib-ee/emqx_ee_bridge

From 275171d2174780808eef8efe967d81065d640281 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 1 Sep 2022 14:37:29 -0300
Subject: [PATCH 123/232] feat: handle lists of servers in mongo servers config

---
 .../src/emqx_connector_mongo.erl              |  32 +++-
 .../test/emqx_connector_mongo_tests.erl       | 168 ++++++++++++++++++
 2 files changed, 198 insertions(+), 2 deletions(-)
 create mode 100644 apps/emqx_connector/test/emqx_connector_mongo_tests.erl

diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index 5bfcbe009..778abf8c2 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -47,6 +47,10 @@
     default_port => ?MONGO_DEFAULT_PORT
 }).
 
+-ifdef(TEST).
+-export([to_servers_raw/1]).
+-endif.
+
 %%=====================================================================
 roots() ->
     [
@@ -447,7 +451,7 @@ may_parse_srv_and_txt_records_(
         true ->
             error({missing_parameter, replica_set_name});
         false ->
-            Config#{hosts => servers_to_bin(Servers)}
+            Config#{hosts => servers_to_bin(lists:flatten(Servers))}
     end;
 may_parse_srv_and_txt_records_(
     #{
@@ -557,9 +561,33 @@ to_servers_raw(Servers) ->
         fun(Server) ->
             emqx_connector_schema_lib:parse_server(Server, ?MONGO_HOST_OPTIONS)
         end,
-        string:tokens(str(Servers), ", ")
+        split_servers(Servers)
     ).
 
+split_servers(L) when is_list(L) ->
+    PossibleTypes = [
+        list(binary()),
+        list(string()),
+        string()
+    ],
+    TypeChecks = lists:map(fun(T) -> typerefl:typecheck(T, L) end, PossibleTypes),
+    case TypeChecks of
+        [ok, _, _] ->
+            %% list(binary())
+            lists:map(fun binary_to_list/1, L);
+        [_, ok, _] ->
+            %% list(string())
+            L;
+        [_, _, ok] ->
+            %% string()
+            string:tokens(L, ", ");
+        [_, _, _] ->
+            %% invalid input
+            throw("List of servers must contain only strings")
+    end;
+split_servers(B) when is_binary(B) ->
+    string:tokens(str(B), ", ").
+
 str(A) when is_atom(A) ->
     atom_to_list(A);
 str(B) when is_binary(B) ->
diff --git a/apps/emqx_connector/test/emqx_connector_mongo_tests.erl b/apps/emqx_connector/test/emqx_connector_mongo_tests.erl
new file mode 100644
index 000000000..7978ed289
--- /dev/null
+++ b/apps/emqx_connector/test/emqx_connector_mongo_tests.erl
@@ -0,0 +1,168 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_connector_mongo_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-define(DEFAULT_MONGO_PORT, 27017).
+
+%%------------------------------------------------------------------------------
+%% Helper fns
+%%------------------------------------------------------------------------------
+
+%%------------------------------------------------------------------------------
+%% Test cases
+%%------------------------------------------------------------------------------
+
+to_servers_raw_test_() ->
+    [
+        {"single server, binary, no port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw(<<"localhost">>)
+                )
+            )},
+        {"single server, string, no port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw("localhost")
+                )
+            )},
+        {"single server, list(binary), no port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw([<<"localhost">>])
+                )
+            )},
+        {"single server, list(string), no port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw(["localhost"])
+                )
+            )},
+        %%%%%%%%%
+        {"single server, binary, with port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", 9999}], emqx_connector_mongo:to_servers_raw(<<"localhost:9999">>)
+                )
+            )},
+        {"single server, string, with port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", 9999}], emqx_connector_mongo:to_servers_raw("localhost:9999")
+                )
+            )},
+        {"single server, list(binary), with port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", 9999}],
+                    emqx_connector_mongo:to_servers_raw([<<"localhost:9999">>])
+                )
+            )},
+        {"single server, list(string), with port",
+            ?_test(
+                ?assertEqual(
+                    [{"localhost", 9999}], emqx_connector_mongo:to_servers_raw(["localhost:9999"])
+                )
+            )},
+        %%%%%%%%%
+        {"multiple servers, string, no port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw("host1, host2")
+                )
+            )},
+        {"multiple servers, binary, no port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw(<<"host1, host2">>)
+                )
+            )},
+        {"multiple servers, list(string), no port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw(["host1", "host2"])
+                )
+            )},
+        {"multiple servers, list(binary), no port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", ?DEFAULT_MONGO_PORT}, {"host2", ?DEFAULT_MONGO_PORT}],
+                    emqx_connector_mongo:to_servers_raw([<<"host1">>, <<"host2">>])
+                )
+            )},
+        %%%%%%%%%
+        {"multiple servers, string, with port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", 1234}, {"host2", 2345}],
+                    emqx_connector_mongo:to_servers_raw("host1:1234, host2:2345")
+                )
+            )},
+        {"multiple servers, binary, with port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", 1234}, {"host2", 2345}],
+                    emqx_connector_mongo:to_servers_raw(<<"host1:1234, host2:2345">>)
+                )
+            )},
+        {"multiple servers, list(string), with port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", 1234}, {"host2", 2345}],
+                    emqx_connector_mongo:to_servers_raw(["host1:1234", "host2:2345"])
+                )
+            )},
+        {"multiple servers, list(binary), with port",
+            ?_test(
+                ?assertEqual(
+                    [{"host1", 1234}, {"host2", 2345}],
+                    emqx_connector_mongo:to_servers_raw([<<"host1:1234">>, <<"host2:2345">>])
+                )
+            )},
+        %%%%%%%%
+        {"multiple servers, invalid list(string)",
+            ?_test(
+                ?assertThrow(
+                    _,
+                    emqx_connector_mongo:to_servers_raw(["host1, host2"])
+                )
+            )},
+        {"multiple servers, invalid list(binary)",
+            ?_test(
+                ?assertThrow(
+                    _,
+                    emqx_connector_mongo:to_servers_raw([<<"host1, host2">>])
+                )
+            )},
+        %% TODO: handle this case??
+        {"multiple servers, mixed list(binary|string)",
+            ?_test(
+                ?assertThrow(
+                    _,
+                    emqx_connector_mongo:to_servers_raw([<<"host1">>, "host2"])
+                )
+            )}
+    ].

From b45f3de8db797f466e0392f4323b63a06d3e00b4 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 2 Sep 2022 12:41:14 +0800
Subject: [PATCH 124/232] refactor(resource): rename metrics batched,queued ->
 batching,queuing

---
 apps/emqx_bridge/i18n/emqx_bridge_schema.conf      |  6 +++---
 apps/emqx_bridge/include/emqx_bridge.hrl           |  8 ++++----
 apps/emqx_bridge/src/emqx_bridge_api.erl           |  6 +++---
 apps/emqx_bridge/src/schema/emqx_bridge_schema.erl |  4 ++--
 apps/emqx_connector/src/emqx_connector_http.erl    | 14 ++++++++++++--
 apps/emqx_resource/src/emqx_resource.erl           |  8 +++++++-
 apps/emqx_resource/src/emqx_resource_manager.erl   |  4 ++--
 apps/emqx_resource/src/emqx_resource_worker.erl    | 12 ++++++------
 .../src/schema/emqx_resource_schema.erl            |  2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl |  2 +-
 10 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
index 06cc41a91..4f079c738 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
@@ -78,7 +78,7 @@ emqx_bridge_schema {
                           }
                   }
 
-    metric_batched {
+    metric_batching {
                    desc {
                          en: """Count of messages that are currently accumulated in memory waiting for sending in one batch."""
                          zh: """当前积压在内存里，等待批量发送的消息个数"""
@@ -161,9 +161,9 @@ emqx_bridge_schema {
                           }
                   }
 
-     metric_queued {
+     metric_queuing {
                    desc {
-                         en: """Count of messages that are currently queued."""
+                         en: """Count of messages that are currently queuing."""
                          zh: """当前被缓存到磁盘队列的消息个数。"""
                         }
                    label: {
diff --git a/apps/emqx_bridge/include/emqx_bridge.hrl b/apps/emqx_bridge/include/emqx_bridge.hrl
index bb8ee6e29..9886a1a62 100644
--- a/apps/emqx_bridge/include/emqx_bridge.hrl
+++ b/apps/emqx_bridge/include/emqx_bridge.hrl
@@ -25,7 +25,7 @@
     Rcvd
 ),
     #{
-        'batched' => Batched,
+        'batching' => Batched,
         'dropped' => Dropped,
         'dropped.other' => DroppedOther,
         'dropped.queue_full' => DroppedQueueFull,
@@ -33,7 +33,7 @@
         'dropped.resource_not_found' => DroppedResourceNotFound,
         'dropped.resource_stopped' => DroppedResourceStopped,
         'matched' => Matched,
-        'queued' => Queued,
+        'queuing' => Queued,
         'sent' => Sent,
         'sent.exception' => SentExcpt,
         'sent.failed' => SentFailed,
@@ -67,7 +67,7 @@
     Rcvd
 ),
     #{
-        'batched' := Batched,
+        'batching' := Batched,
         'dropped' := Dropped,
         'dropped.other' := DroppedOther,
         'dropped.queue_full' := DroppedQueueFull,
@@ -75,7 +75,7 @@
         'dropped.resource_not_found' := DroppedResourceNotFound,
         'dropped.resource_stopped' := DroppedResourceStopped,
         'matched' := Matched,
-        'queued' := Queued,
+        'queuing' := Queued,
         'sent' := Sent,
         'sent.exception' := SentExcpt,
         'sent.failed' := SentFailed,
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index ddffcb79f..9ae6560af 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -206,7 +206,7 @@ info_example_basic(webhook) ->
             worker_pool_size => 1,
             health_check_interval => 15000,
             auto_restart_interval => 15000,
-            query_mode => sync,
+            query_mode => async,
             async_inflight_window => 100,
             enable_queue => true,
             max_queue_bytes => 1024 * 1024 * 1024
@@ -672,7 +672,7 @@ format_resp(
 
 format_metrics(#{
     counters := #{
-        'batched' := Batched,
+        'batching' := Batched,
         'dropped' := Dropped,
         'dropped.other' := DroppedOther,
         'dropped.queue_full' := DroppedQueueFull,
@@ -680,7 +680,7 @@ format_metrics(#{
         'dropped.resource_not_found' := DroppedResourceNotFound,
         'dropped.resource_stopped' := DroppedResourceStopped,
         'matched' := Matched,
-        'queued' := Queued,
+        'queuing' := Queued,
         'sent' := Sent,
         'sent.exception' := SentExcpt,
         'sent.failed' := SentFailed,
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index f55ac840e..9180ec38a 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -102,7 +102,7 @@ fields(bridges) ->
     ] ++ ee_fields_bridges();
 fields("metrics") ->
     [
-        {"batched", mk(integer(), #{desc => ?DESC("metric_batched")})},
+        {"batching", mk(integer(), #{desc => ?DESC("metric_batching")})},
         {"dropped", mk(integer(), #{desc => ?DESC("metric_dropped")})},
         {"dropped.other", mk(integer(), #{desc => ?DESC("metric_dropped_other")})},
         {"dropped.queue_full", mk(integer(), #{desc => ?DESC("metric_dropped_queue_full")})},
@@ -113,7 +113,7 @@ fields("metrics") ->
         {"dropped.resource_stopped",
             mk(integer(), #{desc => ?DESC("metric_dropped_resource_stopped")})},
         {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})},
-        {"queued", mk(integer(), #{desc => ?DESC("metric_queued")})},
+        {"queuing", mk(integer(), #{desc => ?DESC("metric_queuing")})},
         {"sent", mk(integer(), #{desc => ?DESC("metric_sent")})},
         {"sent.exception", mk(integer(), #{desc => ?DESC("metric_sent_exception")})},
         {"sent.failed", mk(integer(), #{desc => ?DESC("metric_sent_failed")})},
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 3eb55c9a4..5f33714ef 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -31,7 +31,8 @@
     on_stop/2,
     on_query/3,
     on_query_async/4,
-    on_get_status/2
+    on_get_status/2,
+    reply_delegator/2
 ]).
 
 -type url() :: emqx_http_lib:uri_map().
@@ -378,9 +379,18 @@ on_query_async(
         Method,
         NRequest,
         Timeout,
-        ReplyFunAndArgs
+        {fun ?MODULE:reply_delegator/2, [ReplyFunAndArgs]}
     ).
 
+reply_delegator(ReplyFunAndArgs, Result) ->
+    case Result of
+        {error, Reason} when Reason =:= econnrefused; Reason =:= timeout ->
+            Result1 = {error, {recoverable_error, Reason}},
+            emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1);
+        _ ->
+            emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result)
+    end.
+
 on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) ->
     case do_get_status(PoolName, Timeout) of
         true ->
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index c1d500e8b..309c34195 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -110,7 +110,7 @@
     list_group_instances/1
 ]).
 
--export([inc_received/1]).
+-export([inc_received/1, apply_reply_fun/2]).
 
 -optional_callbacks([
     on_query/3,
@@ -441,6 +441,12 @@ check_and_do(ResourceType, RawConfig, Do) when is_function(Do) ->
         Error -> Error
     end.
 
+apply_reply_fun({F, A}, Result) when is_function(F) ->
+    _ = erlang:apply(F, A ++ [Result]),
+    ok;
+apply_reply_fun(From, Result) ->
+    gen_server:reply(From, Result).
+
 %% =================================================================================
 
 inc_received(ResId) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 2581a3001..b12fc3a44 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -132,8 +132,8 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
             'matched',
             'sent',
             'dropped',
-            'queued',
-            'batched',
+            'queuing',
+            'batching',
             'sent.success',
             'sent.failed',
             'sent.exception',
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index a0632a761..4de8bd6ef 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -131,7 +131,7 @@ init({Id, Index, Opts}) ->
             false ->
                 undefined
         end,
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', queue_count(Queue)),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', queue_count(Queue)),
     ok = inflight_new(Name),
     St = #{
         id => Id,
@@ -273,12 +273,12 @@ retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = S
 drop_head(Id, Q) ->
     {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
     ok = replayq:ack(Q1, AckRef),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -1),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -1),
     Q1.
 
 query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left, id := Id} = St0) ->
     Acc1 = [?QUERY(From, Request) | Acc],
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batched'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batching'),
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
         true -> flush(St);
@@ -313,7 +313,7 @@ flush(
         inflight_name => maps:get(name, St),
         inflight_window => maps:get(async_inflight_window, St)
     },
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batched', -length(Batch)),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batching', -length(Batch)),
     Result = call_query(configured, Id, Batch, QueryOpts),
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
     case batch_reply_caller(Id, Result, Batch) of
@@ -338,13 +338,13 @@ maybe_append_queue(Id, Q, Items) ->
                 {Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts),
                 ok = replayq:ack(Q1, QAckRef),
                 Dropped = length(Items2),
-                emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped),
+                emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -Dropped),
                 emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
                 emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
                 ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
                 Q1
         end,
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
     replayq:append(Q2, Items).
 
 batch_reply_caller(Id, BatchResult, Batch) ->
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index 9e54c8a7b..df284bbe8 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -76,7 +76,7 @@ auto_restart_interval(_) -> undefined.
 
 query_mode(type) -> enum([sync, async]);
 query_mode(desc) -> ?DESC("query_mode");
-query_mode(default) -> sync;
+query_mode(default) -> async;
 query_mode(required) -> false;
 query_mode(_) -> undefined.
 
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index f76eb388e..616842292 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -61,7 +61,7 @@ values(post) ->
             enable_batch => false,
             batch_size => ?DEFAULT_BATCH_SIZE,
             batch_time => ?DEFAULT_BATCH_TIME,
-            query_mode => sync,
+            query_mode => async,
             enable_queue => false,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
         }

From 83f21b4c6502f384ec8047dcfcd2d8f551cd009a Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 2 Sep 2022 12:41:54 +0800
Subject: [PATCH 125/232] refactor(resource): remove metrics 'sent.exception'

---
 apps/emqx_bridge/i18n/emqx_bridge_schema.conf      | 10 ----------
 apps/emqx_bridge/include/emqx_bridge.hrl           |  6 +-----
 apps/emqx_bridge/src/emqx_bridge_api.erl           |  9 +++------
 apps/emqx_bridge/src/schema/emqx_bridge_schema.erl |  1 -
 apps/emqx_resource/src/emqx_resource_manager.erl   |  1 -
 apps/emqx_resource/src/emqx_resource_worker.erl    |  2 +-
 6 files changed, 5 insertions(+), 24 deletions(-)

diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
index 4f079c738..de9e10e74 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
@@ -181,16 +181,6 @@ emqx_bridge_schema {
                            zh: "已发送"
                           }
                   }
-     metric_sent_exception {
-                   desc {
-                         en: """Count of messages that were sent but exceptions occur."""
-                         zh: """发送出现异常的消息个数。"""
-                        }
-                   label: {
-                           en: "Sent Exception"
-                           zh: "发送异常"
-                          }
-                  }
 
      metric_sent_failed {
                    desc {
diff --git a/apps/emqx_bridge/include/emqx_bridge.hrl b/apps/emqx_bridge/include/emqx_bridge.hrl
index 9886a1a62..2b64dba70 100644
--- a/apps/emqx_bridge/include/emqx_bridge.hrl
+++ b/apps/emqx_bridge/include/emqx_bridge.hrl
@@ -1,6 +1,6 @@
 -define(EMPTY_METRICS,
     ?METRICS(
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
     )
 ).
 
@@ -15,7 +15,6 @@
     Matched,
     Queued,
     Sent,
-    SentExcpt,
     SentFailed,
     SentInflight,
     SentSucc,
@@ -35,7 +34,6 @@
         'matched' => Matched,
         'queuing' => Queued,
         'sent' => Sent,
-        'sent.exception' => SentExcpt,
         'sent.failed' => SentFailed,
         'sent.inflight' => SentInflight,
         'sent.success' => SentSucc,
@@ -57,7 +55,6 @@
     Matched,
     Queued,
     Sent,
-    SentExcpt,
     SentFailed,
     SentInflight,
     SentSucc,
@@ -77,7 +74,6 @@
         'matched' := Matched,
         'queuing' := Queued,
         'sent' := Sent,
-        'sent.exception' := SentExcpt,
         'sent.failed' := SentFailed,
         'sent.inflight' := SentInflight,
         'sent.success' := SentSucc,
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 9ae6560af..0446c8069 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -617,11 +617,11 @@ aggregate_metrics(AllMetrics) ->
         fun(
             #{
                 metrics := ?metrics(
-                    M1, M2, M3, M4, M5, M6, M7, M8, M9, M10, M11, M12, M13, M14, M15, M16, M17, M18
+                    M1, M2, M3, M4, M5, M6, M7, M8, M9, M10, M11, M12, M13, M14, M15, M16, M17
                 )
             },
             ?metrics(
-                N1, N2, N3, N4, N5, N6, N7, N8, N9, N10, N11, N12, N13, N14, N15, N16, N17, N18
+                N1, N2, N3, N4, N5, N6, N7, N8, N9, N10, N11, N12, N13, N14, N15, N16, N17
             )
         ) ->
             ?METRICS(
@@ -641,8 +641,7 @@ aggregate_metrics(AllMetrics) ->
                 M14 + N14,
                 M15 + N15,
                 M16 + N16,
-                M17 + N17,
-                M18 + N18
+                M17 + N17
             )
         end,
         InitMetrics,
@@ -682,7 +681,6 @@ format_metrics(#{
         'matched' := Matched,
         'queuing' := Queued,
         'sent' := Sent,
-        'sent.exception' := SentExcpt,
         'sent.failed' := SentFailed,
         'sent.inflight' := SentInflight,
         'sent.success' := SentSucc,
@@ -703,7 +701,6 @@ format_metrics(#{
         Matched,
         Queued,
         Sent,
-        SentExcpt,
         SentFailed,
         SentInflight,
         SentSucc,
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 9180ec38a..1a20da6d6 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -115,7 +115,6 @@ fields("metrics") ->
         {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})},
         {"queuing", mk(integer(), #{desc => ?DESC("metric_queuing")})},
         {"sent", mk(integer(), #{desc => ?DESC("metric_sent")})},
-        {"sent.exception", mk(integer(), #{desc => ?DESC("metric_sent_exception")})},
         {"sent.failed", mk(integer(), #{desc => ?DESC("metric_sent_failed")})},
         {"sent.inflight", mk(integer(), #{desc => ?DESC("metric_sent_inflight")})},
         {"sent.success", mk(integer(), #{desc => ?DESC("metric_sent_success")})},
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index b12fc3a44..c8c097c30 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -136,7 +136,6 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
             'batching',
             'sent.success',
             'sent.failed',
-            'sent.exception',
             'sent.inflight',
             'dropped.queue_not_enabled',
             'dropped.queue_full',
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 4de8bd6ef..a0c7ebcdb 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -375,7 +375,7 @@ reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
     handle_query_result(Id, Result, BlockWorker).
 
 handle_query_result(Id, ?RESOURCE_ERROR_M(exception, _), BlockWorker) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.exception'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
 handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _) when
     NotWorking == not_connected; NotWorking == blocked

From 98cc018a88b1c29444f2a3a031aeab2baff98396 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Fri, 2 Sep 2022 13:51:36 +0800
Subject: [PATCH 126/232] chore: update dashboard for e5.0

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 63e18053f..d0a502a27 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v1.0.7
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.1
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.2
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

From 26234d38b91d38ebbffce3aeba1c4d2a0af96fdd Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 2 Sep 2022 18:36:38 +0800
Subject: [PATCH 127/232] fix: mark the async msg 'queuing' not 'sent.inflight'
 on recoverable_error

---
 .../src/emqx_connector_http.erl               | 18 +++---
 .../src/emqx_resource_worker.erl              | 56 ++++++++++---------
 .../test/emqx_connector_demo.erl              |  2 +-
 3 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 5f33714ef..ed5897a59 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -382,15 +382,6 @@ on_query_async(
         {fun ?MODULE:reply_delegator/2, [ReplyFunAndArgs]}
     ).
 
-reply_delegator(ReplyFunAndArgs, Result) ->
-    case Result of
-        {error, Reason} when Reason =:= econnrefused; Reason =:= timeout ->
-            Result1 = {error, {recoverable_error, Reason}},
-            emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1);
-        _ ->
-            emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result)
-    end.
-
 on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) ->
     case do_get_status(PoolName, Timeout) of
         true ->
@@ -544,3 +535,12 @@ bin(Str) when is_list(Str) ->
     list_to_binary(Str);
 bin(Atom) when is_atom(Atom) ->
     atom_to_binary(Atom, utf8).
+
+reply_delegator(ReplyFunAndArgs, Result) ->
+    case Result of
+        {error, Reason} when Reason =:= econnrefused; Reason =:= timeout ->
+            Result1 = {error, {recoverable_error, Reason}},
+            emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1);
+        _ ->
+            emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result)
+    end.
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index a0c7ebcdb..c5841b7ef 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -154,12 +154,12 @@ running(enter, _, _St) ->
 running(cast, resume, _St) ->
     keep_state_and_data;
 running(cast, block, St) ->
-    {next_state, block, St};
+    {next_state, blocked, St};
 running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{id := Id, queue := Q} = St) when
     is_list(Batch)
 ->
     Q1 = maybe_append_queue(Id, Q, [?Q_ITEM(Query) || Query <- Batch]),
-    {next_state, block, St#{queue := Q1}};
+    {next_state, blocked, St#{queue := Q1}};
 running({call, From}, {query, Request, _Opts}, St) ->
     query_or_acc(From, Request, St);
 running(cast, {query, Request, Opts}, St) ->
@@ -366,7 +366,7 @@ reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) ->
 reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) ->
     _ =
         case Result of
-            {async_return, _} -> ok;
+            {async_return, _} -> no_reply_for_now;
             _ -> apply(ReplyFun, Args ++ [Result])
         end,
     handle_query_result(Id, Result, BlockWorker);
@@ -395,6 +395,9 @@ handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.other'),
     BlockWorker;
 handle_query_result(Id, {error, {recoverable_error, _}}, _BlockWorker) ->
+    %% the message will be queued in replayq or inflight window,
+    %% i.e. the counter 'queuing' will increase, so we pretend that we have not
+    %% sent this message.
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', -1),
     true;
 handle_query_result(Id, {error, _}, BlockWorker) ->
@@ -450,15 +453,10 @@ call_query(QM0, Id, Query, QueryOpts) ->
 apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
-    ?APPLY_RESOURCE(
-        begin
-            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
-            Result = Mod:on_query(Id, Request, ResSt),
-            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
-            Result
-        end,
-        Request
-    );
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
+    Result = ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
+    Result;
 apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
@@ -483,16 +481,12 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
 apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
     ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', length(Batch)),
-    ?APPLY_RESOURCE(
-        begin
-            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
-            Result = Mod:on_batch_query(Id, Requests, ResSt),
-            ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
-            Result
-        end,
-        Batch
-    );
+    BatchLen = length(Batch),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', BatchLen),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', BatchLen),
+    Result = ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch),
+    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -BatchLen),
+    Result;
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
@@ -503,8 +497,9 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
                 ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', length(Batch)),
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
+                BatchLen = length(Batch),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', BatchLen),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', BatchLen),
                 ReplyFun = fun ?MODULE:batch_reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = {ReplyFun, [self(), Id, Name, Ref, Batch]},
@@ -517,20 +512,29 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ).
 
 reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request), Result) ->
+    %% NOTE: 'sent.inflight' is message count that sent but no ACK received,
+    %%        NOT the message number ququed in the inflight window.
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
     case reply_caller(Id, ?REPLY(From, Request, Result)) of
         true ->
+            %% we marked these messages are 'queuing' although they are in inflight window
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
             ?MODULE:block(Pid);
         false ->
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
             inflight_drop(Name, Ref)
     end.
 
 batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
+    %% NOTE: 'sent.inflight' is message count that sent but no ACK received,
+    %%        NOT the message number ququed in the inflight window.
+    BatchLen = length(Batch),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -BatchLen),
     case batch_reply_caller(Id, Result, Batch) of
         true ->
+            %% we marked these messages are 'queuing' although they are in inflight window
+            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', BatchLen),
             ?MODULE:block(Pid);
         false ->
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -length(Batch)),
             inflight_drop(Name, Ref)
     end.
 %%==============================================================================
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index 4999b9410..a645af7d8 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -96,7 +96,7 @@ on_query(_InstId, {inc_counter, N}, #{pid := Pid}) ->
     Pid ! {From, {inc, N}},
     receive
         {ReqRef, ok} -> ok;
-        {ReqRef, incorrect_status} -> {recoverable_error, incorrect_status}
+        {ReqRef, incorrect_status} -> {error, {recoverable_error, incorrect_status}}
     after 1000 ->
         {error, timeout}
     end;

From f1048babd882b1b037fe6c3b417d0dc08bb703a3 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 2 Sep 2022 11:05:24 -0300
Subject: [PATCH 128/232] test: refactor to use hocon and schema

---
 .../test/emqx_ee_bridge_mongodb_SUITE.erl     | 191 ++++++++++--------
 1 file changed, 106 insertions(+), 85 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
index 775acf7b0..bc8c0b04f 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
@@ -40,15 +40,16 @@ init_per_group(Type = rs, Config) ->
     MongoPort = list_to_integer(os:getenv("MONGO_RS_PORT", "27017")),
     case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
         true ->
-            _ = application:load(emqx_ee_bridge),
-            _ = emqx_ee_bridge:module_info(),
+            ensure_loaded(),
             ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
             emqx_mgmt_api_test_util:init_suite(),
-            MongoConfig = mongo_config(MongoHost, MongoPort, Type),
+            {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type),
             [
                 {mongo_host, MongoHost},
                 {mongo_port, MongoPort},
-                {mongo_config, MongoConfig}
+                {mongo_config, MongoConfig},
+                {mongo_type, Type},
+                {mongo_name, Name}
                 | Config
             ];
         false ->
@@ -59,15 +60,16 @@ init_per_group(Type = sharded, Config) ->
     MongoPort = list_to_integer(os:getenv("MONGO_SHARDED_PORT", "27017")),
     case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
         true ->
-            _ = application:load(emqx_ee_bridge),
-            _ = emqx_ee_bridge:module_info(),
+            ensure_loaded(),
             ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
             emqx_mgmt_api_test_util:init_suite(),
-            MongoConfig = mongo_config(MongoHost, MongoPort, Type),
+            {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type),
             [
                 {mongo_host, MongoHost},
                 {mongo_port, MongoPort},
-                {mongo_config, MongoConfig}
+                {mongo_config, MongoConfig},
+                {mongo_type, Type},
+                {mongo_name, Name}
                 | Config
             ];
         false ->
@@ -78,15 +80,16 @@ init_per_group(Type = single, Config) ->
     MongoPort = list_to_integer(os:getenv("MONGO_SINGLE_PORT", "27017")),
     case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
         true ->
-            _ = application:load(emqx_ee_bridge),
-            _ = emqx_ee_bridge:module_info(),
+            ensure_loaded(),
             ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
             emqx_mgmt_api_test_util:init_suite(),
-            MongoConfig = mongo_config(MongoHost, MongoPort, Type),
+            {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type),
             [
                 {mongo_host, MongoHost},
                 {mongo_port, MongoPort},
-                {mongo_config, MongoConfig}
+                {mongo_config, MongoConfig},
+                {mongo_type, Type},
+                {mongo_name, Name}
                 | Config
             ];
         false ->
@@ -118,65 +121,84 @@ end_per_testcase(_Testcase, Config) ->
 %% Helper fns
 %%------------------------------------------------------------------------------
 
-mongo_config(MongoHost0, MongoPort0, rs) ->
-    MongoHost = list_to_binary(MongoHost0),
-    MongoPort = integer_to_binary(MongoPort0),
-    Servers = <<MongoHost/binary, ":", MongoPort/binary>>,
-    Name = atom_to_binary(?MODULE),
-    #{
-        <<"type">> => <<"mongodb_rs">>,
-        <<"name">> => Name,
-        <<"enable">> => true,
-        <<"collection">> => <<"mycol">>,
-        <<"servers">> => Servers,
-        <<"database">> => <<"mqtt">>,
-        <<"w_mode">> => <<"safe">>,
-        <<"replica_set_name">> => <<"rs0">>
-    };
-mongo_config(MongoHost0, MongoPort0, sharded) ->
-    MongoHost = list_to_binary(MongoHost0),
-    MongoPort = integer_to_binary(MongoPort0),
-    Servers = <<MongoHost/binary, ":", MongoPort/binary>>,
-    Name = atom_to_binary(?MODULE),
-    #{
-        <<"type">> => <<"mongodb_sharded">>,
-        <<"name">> => Name,
-        <<"enable">> => true,
-        <<"collection">> => <<"mycol">>,
-        <<"servers">> => Servers,
-        <<"database">> => <<"mqtt">>,
-        <<"w_mode">> => <<"safe">>
-    };
-mongo_config(MongoHost0, MongoPort0, single) ->
-    MongoHost = list_to_binary(MongoHost0),
-    MongoPort = integer_to_binary(MongoPort0),
-    Server = <<MongoHost/binary, ":", MongoPort/binary>>,
-    Name = atom_to_binary(?MODULE),
-    #{
-        <<"type">> => <<"mongodb_single">>,
-        <<"name">> => Name,
-        <<"enable">> => true,
-        <<"collection">> => <<"mycol">>,
-        <<"server">> => Server,
-        <<"database">> => <<"mqtt">>,
-        <<"w_mode">> => <<"safe">>
-    }.
+ensure_loaded() ->
+    _ = application:load(emqx_ee_bridge),
+    _ = emqx_ee_bridge:module_info(),
+    ok.
 
-create_bridge(Config0 = #{<<"type">> := Type, <<"name">> := Name}) ->
-    Config = maps:without(
-        [
-            <<"type">>,
-            <<"name">>
-        ],
-        Config0
-    ),
-    emqx_bridge:create(Type, Name, Config).
+mongo_type_bin(rs) ->
+    <<"mongodb_rs">>;
+mongo_type_bin(sharded) ->
+    <<"mongodb_sharded">>;
+mongo_type_bin(single) ->
+    <<"mongodb_single">>.
+
+mongo_config(MongoHost, MongoPort0, rs = Type) ->
+    MongoPort = integer_to_list(MongoPort0),
+    Servers = MongoHost ++ ":" ++ MongoPort,
+    Name = atom_to_binary(?MODULE),
+    ConfigString =
+        io_lib:format(
+            "bridges.mongodb_rs.~s {\n"
+            "  enable = true\n"
+            "  collection = mycol\n"
+            "  replica_set_name = rs0\n"
+            "  servers = [~p]\n"
+            "  w_mode = safe\n"
+            "  database = mqtt\n"
+            "}",
+            [Name, Servers]
+        ),
+    {Name, parse_and_check(ConfigString, Type, Name)};
+mongo_config(MongoHost, MongoPort0, sharded = Type) ->
+    MongoPort = integer_to_list(MongoPort0),
+    Servers = MongoHost ++ ":" ++ MongoPort,
+    Name = atom_to_binary(?MODULE),
+    ConfigString =
+        io_lib:format(
+            "bridges.mongodb_sharded.~s {\n"
+            "  enable = true\n"
+            "  collection = mycol\n"
+            "  servers = [~p]\n"
+            "  w_mode = safe\n"
+            "  database = mqtt\n"
+            "}",
+            [Name, Servers]
+        ),
+    {Name, parse_and_check(ConfigString, Type, Name)};
+mongo_config(MongoHost, MongoPort0, single = Type) ->
+    MongoPort = integer_to_list(MongoPort0),
+    Server = MongoHost ++ ":" ++ MongoPort,
+    Name = atom_to_binary(?MODULE),
+    ConfigString =
+        io_lib:format(
+            "bridges.mongodb_single.~s {\n"
+            "  enable = true\n"
+            "  collection = mycol\n"
+            "  server = ~p\n"
+            "  w_mode = safe\n"
+            "  database = mqtt\n"
+            "}",
+            [Name, Server]
+        ),
+    {Name, parse_and_check(ConfigString, Type, Name)}.
+
+parse_and_check(ConfigString, Type, Name) ->
+    {ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
+    TypeBin = mongo_type_bin(Type),
+    hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
+    #{<<"bridges">> := #{TypeBin := #{Name := Config}}} = RawConf,
+    Config.
+
+create_bridge(Config) ->
+    Type = mongo_type_bin(?config(mongo_type, Config)),
+    Name = ?config(mongo_name, Config),
+    MongoConfig = ?config(mongo_config, Config),
+    emqx_bridge:create(Type, Name, MongoConfig).
 
 delete_bridge(Config) ->
-    #{
-        <<"type">> := Type,
-        <<"name">> := Name
-    } = ?config(mongo_config, Config),
+    Type = mongo_type_bin(?config(mongo_type, Config)),
+    Name = ?config(mongo_name, Config),
     emqx_bridge:remove(Type, Name).
 
 create_bridge_http(Params) ->
@@ -188,11 +210,9 @@ create_bridge_http(Params) ->
     end.
 
 clear_db(Config) ->
-    #{
-        <<"name">> := Name,
-        <<"type">> := Type,
-        <<"collection">> := Collection
-    } = ?config(mongo_config, Config),
+    Type = mongo_type_bin(?config(mongo_type, Config)),
+    Name = ?config(mongo_name, Config),
+    #{<<"collection">> := Collection} = ?config(mongo_config, Config),
     ResourceID = emqx_bridge_resource:resource_id(Type, Name),
     {ok, _, #{state := #{poolname := PoolName}}} = emqx_resource:get_instance(ResourceID),
     Selector = #{},
@@ -202,19 +222,15 @@ clear_db(Config) ->
     ok.
 
 find_all(Config) ->
-    #{
-        <<"name">> := Name,
-        <<"type">> := Type,
-        <<"collection">> := Collection
-    } = ?config(mongo_config, Config),
+    Type = mongo_type_bin(?config(mongo_type, Config)),
+    Name = ?config(mongo_name, Config),
+    #{<<"collection">> := Collection} = ?config(mongo_config, Config),
     ResourceID = emqx_bridge_resource:resource_id(Type, Name),
     emqx_resource:query(ResourceID, {find, Collection, #{}, #{}}).
 
 send_message(Config, Payload) ->
-    #{
-        <<"name">> := Name,
-        <<"type">> := Type
-    } = ?config(mongo_config, Config),
+    Name = ?config(mongo_name, Config),
+    Type = mongo_type_bin(?config(mongo_type, Config)),
     BridgeID = emqx_bridge_resource:bridge_id(Type, Name),
     emqx_bridge:send_message(BridgeID, Payload).
 
@@ -223,10 +239,9 @@ send_message(Config, Payload) ->
 %%------------------------------------------------------------------------------
 
 t_setup_via_config_and_publish(Config) ->
-    MongoConfig = ?config(mongo_config, Config),
     ?assertMatch(
         {ok, _},
-        create_bridge(MongoConfig)
+        create_bridge(Config)
     ),
     Val = erlang:unique_integer(),
     ok = send_message(Config, #{key => Val}),
@@ -237,7 +252,13 @@ t_setup_via_config_and_publish(Config) ->
     ok.
 
 t_setup_via_http_api_and_publish(Config) ->
-    MongoConfig = ?config(mongo_config, Config),
+    Type = mongo_type_bin(?config(mongo_type, Config)),
+    Name = ?config(mongo_name, Config),
+    MongoConfig0 = ?config(mongo_config, Config),
+    MongoConfig = MongoConfig0#{
+        <<"name">> => Name,
+        <<"type">> => Type
+    },
     ?assertMatch(
         {ok, _},
         create_bridge_http(MongoConfig)

From c695e67e18ea4247ca699c233f5769b374c6e899 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Sat, 3 Sep 2022 12:19:05 +0800
Subject: [PATCH 129/232] chore: release e5.0.0-beta.3

---
 apps/emqx/include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index b6b601f0e..542061946 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -35,7 +35,7 @@
 -define(EMQX_RELEASE_CE, "5.0.6").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.0.0-beta.2").
+-define(EMQX_RELEASE_EE, "5.0.0-beta.3").
 
 %% the HTTP API version
 -define(EMQX_API_VERSION, "5.0").

From d4476593653df4cf5982f3012eaf0176a2b99d2a Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 5 Sep 2022 11:29:49 +0800
Subject: [PATCH 130/232] feat: add edition info to /nodes api

---
 apps/emqx_machine/src/emqx_restricted_shell.erl         | 5 +++--
 apps/emqx_management/src/emqx_mgmt.erl                  | 7 +++++++
 apps/emqx_management/src/emqx_mgmt_api_nodes.erl        | 7 ++++++-
 apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl | 5 +++++
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_machine/src/emqx_restricted_shell.erl b/apps/emqx_machine/src/emqx_restricted_shell.erl
index cc475cb56..31ee16986 100644
--- a/apps/emqx_machine/src/emqx_restricted_shell.erl
+++ b/apps/emqx_machine/src/emqx_restricted_shell.erl
@@ -45,9 +45,10 @@ set_prompt_func() ->
 prompt_func(PropList) ->
     Line = proplists:get_value(history, PropList, 1),
     Version = emqx_release:version(),
+    Edition = emqx_release:edition(),
     case is_alive() of
-        true -> io_lib:format(<<"~ts(~s)~w> ">>, [Version, node(), Line]);
-        false -> io_lib:format(<<"~ts ~w> ">>, [Version, Line])
+        true -> io_lib:format(<<"~ts-~ts(~s)~w> ">>, [Edition, Version, node(), Line]);
+        false -> io_lib:format(<<"~ts-~ts ~w> ">>, [Edition, Version, Line])
     end.
 
 local_allowed(MF, Args, State) ->
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 8f73d5767..87422697a 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -141,6 +141,7 @@ node_info() ->
         node_status => 'running',
         uptime => proplists:get_value(uptime, BrokerInfo),
         version => iolist_to_binary(proplists:get_value(version, BrokerInfo)),
+        edition => edition(),
         role => mria_rlog:role()
     }.
 
@@ -553,3 +554,9 @@ max_row_limit() ->
     ?MAX_ROW_LIMIT.
 
 table_size(Tab) -> ets:info(Tab, size).
+
+edition() ->
+    case emqx_release:edition() of
+        ee -> <<"enterprise">>;
+        ce -> <<"community">>
+    end.
diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
index e0f0912df..50e65f581 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
@@ -215,7 +215,12 @@ fields(node_info) ->
         {version,
             mk(
                 string(),
-                #{desc => <<"Release version">>, example => "5.0.0-beat.3-00000000"}
+                #{desc => <<"Release version">>, example => "5.0.0"}
+            )},
+        {edition,
+            mk(
+                enum([community, enterprise]),
+                #{desc => <<"Release edition">>, example => "community"}
             )},
         {sys_path,
             mk(
diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
index 4fb512ed0..e1a90f6cc 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
@@ -57,6 +57,11 @@ t_nodes_api(_) ->
     LocalNodeInfo = hd(NodesResponse),
     Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8),
     ?assertEqual(Node, node()),
+    Edition = maps:get(<<"edition">>, LocalNodeInfo),
+    case emqx_release:edition() of
+        ee -> ?assertEqual(<<"enterprise">>, Edition);
+        ce -> ?assertEqual(<<"community">>, Edition)
+    end,
 
     NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]),
     {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath),

From 57cc880977ae414934b9133be8db208b33b1cd8b Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 5 Sep 2022 14:26:53 +0800
Subject: [PATCH 131/232] chore: change community to opensource

---
 apps/emqx_machine/src/emqx_machine.app.src              | 2 +-
 apps/emqx_management/src/emqx_mgmt.erl                  | 2 +-
 apps/emqx_management/src/emqx_mgmt_api_nodes.erl        | 4 ++--
 apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src
index 59a5ad4b8..63c6c01ad 100644
--- a/apps/emqx_machine/src/emqx_machine.app.src
+++ b/apps/emqx_machine/src/emqx_machine.app.src
@@ -3,7 +3,7 @@
     {id, "emqx_machine"},
     {description, "The EMQX Machine"},
     % strict semver, bump manually!
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {modules, []},
     {registered, []},
     {applications, [kernel, stdlib]},
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 87422697a..7116a8c5d 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -558,5 +558,5 @@ table_size(Tab) -> ets:info(Tab, size).
 edition() ->
     case emqx_release:edition() of
         ee -> <<"enterprise">>;
-        ce -> <<"community">>
+        ce -> <<"opensource">>
     end.
diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
index 50e65f581..c58166b1a 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
@@ -219,8 +219,8 @@ fields(node_info) ->
             )},
         {edition,
             mk(
-                enum([community, enterprise]),
-                #{desc => <<"Release edition">>, example => "community"}
+                enum([opensource, enterprise]),
+                #{desc => <<"Release edition">>, example => "opensource"}
             )},
         {sys_path,
             mk(
diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
index e1a90f6cc..d031b3d09 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
@@ -60,7 +60,7 @@ t_nodes_api(_) ->
     Edition = maps:get(<<"edition">>, LocalNodeInfo),
     case emqx_release:edition() of
         ee -> ?assertEqual(<<"enterprise">>, Edition);
-        ce -> ?assertEqual(<<"community">>, Edition)
+        ce -> ?assertEqual(<<"opensource">>, Edition)
     end,
 
     NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]),

From d73f22181d91d19a9c0a445ebd5ebfe0aa923c3d Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 5 Sep 2022 15:02:50 +0800
Subject: [PATCH 132/232] chore: fix dialyzer warning.

---
 apps/emqx_management/src/emqx_mgmt.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 7116a8c5d..eefee0dfc 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -19,6 +19,7 @@
 -include("emqx_mgmt.hrl").
 -elvis([{elvis_style, invalid_dynamic_call, disable}]).
 -elvis([{elvis_style, god_modules, disable}]).
+-dialyzer({nowarn_function, edition/0}).
 
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("emqx/include/emqx.hrl").

From 60a90858f8745fbe1c46154007e805f69b9a2c64 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 6 Sep 2022 15:34:23 +0800
Subject: [PATCH 133/232] feat: check dependent actions before removing the
 bridges

---
 apps/emqx_bridge/src/emqx_bridge.erl          |  19 ++++
 apps/emqx_bridge/src/emqx_bridge_api.erl      |  26 ++++-
 .../test/emqx_bridge_api_SUITE.erl            |  82 +++++++++++++-
 .../test/emqx_dashboard_api_test_helpers.erl  |   2 +-
 .../emqx_rule_engine/src/emqx_rule_engine.erl |  61 +++++++++++
 .../src/emqx_rule_sqltester.erl               |   8 +-
 .../test/emqx_rule_engine_SUITE.erl           | 103 +++++++++++++++++-
 7 files changed, 286 insertions(+), 15 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index 151275bd9..d4d24ef3a 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -37,6 +37,7 @@
     create/3,
     disable_enable/3,
     remove/2,
+    check_deps_and_remove/3,
     list/0
 ]).
 
@@ -247,6 +248,24 @@ remove(BridgeType, BridgeName) ->
         #{override_to => cluster}
     ).
 
+check_deps_and_remove(BridgeType, BridgeName, RemoveDeps) ->
+    Id = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
+    %% NOTE: This violates the design: Rule depends on data-bridge but not vice versa.
+    case emqx_rule_engine:get_rule_ids_by_action(Id) of
+        [] ->
+            remove(BridgeType, BridgeName);
+        Rules when RemoveDeps =:= false ->
+            {error, {rules_deps_on_this_bridge, Rules}};
+        Rules when RemoveDeps =:= true ->
+            lists:foreach(
+                fun(R) ->
+                    emqx_rule_engine:ensure_action_removed(R, Id)
+                end,
+                Rules
+            ),
+            remove(BridgeType, BridgeName)
+    end.
+
 %%========================================================================================
 %% Helper functions
 %%========================================================================================
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index aa6a6af89..df5121f67 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -331,6 +331,7 @@ schema("/bridges/:id") ->
             responses => #{
                 204 => <<"Bridge deleted">>,
                 400 => error_schema(['INVALID_ID'], "Update bridge failed"),
+                403 => error_schema('FORBIDDEN_REQUEST', "forbidden operation"),
                 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable")
             }
         }
@@ -424,13 +425,28 @@ schema("/nodes/:node/bridges/:id/operation/:operation") ->
                 {404, error_msg('NOT_FOUND', <<"bridge not found">>)}
         end
     );
-'/bridges/:id'(delete, #{bindings := #{id := Id}}) ->
+'/bridges/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) ->
+    AlsoDeleteActs =
+        case maps:get(<<"also_delete_dep_actions">>, Qs, <<"false">>) of
+            <<"true">> -> true;
+            true -> true;
+            _ -> false
+        end,
     ?TRY_PARSE_ID(
         Id,
-        case emqx_bridge:remove(BridgeType, BridgeName) of
-            {ok, _} -> {204};
-            {error, timeout} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
-            {error, Reason} -> {500, error_msg('INTERNAL_ERROR', Reason)}
+        case emqx_bridge:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActs) of
+            {ok, _} ->
+                204;
+            {error, {rules_deps_on_this_bridge, RuleIds}} ->
+                {403,
+                    error_msg(
+                        'FORBIDDEN_REQUEST',
+                        {<<"There're some rules dependent on this bridge">>, RuleIds}
+                    )};
+            {error, timeout} ->
+                {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+            {error, Reason} ->
+                {500, error_msg('INTERNAL_ERROR', Reason)}
         end
     ).
 
diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
index 9346fb9c0..c0a58abcc 100644
--- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
@@ -61,14 +61,18 @@ init_per_suite(Config) ->
     _ = application:stop(emqx_resource),
     _ = application:stop(emqx_connector),
     ok = emqx_common_test_helpers:start_apps(
-        [emqx_bridge, emqx_dashboard],
+        [emqx_rule_engine, emqx_bridge, emqx_dashboard],
         fun set_special_configs/1
     ),
+    ok = emqx_common_test_helpers:load_config(
+        emqx_rule_engine_schema,
+        <<"rule_engine {rules {}}">>
+    ),
     ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?CONF_DEFAULT),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_dashboard]),
+    emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_bridge, emqx_dashboard]),
     ok.
 
 set_special_configs(emqx_dashboard) ->
@@ -301,6 +305,80 @@ t_http_crud_apis(Config) ->
     ),
     ok.
 
+t_check_dependent_actions_on_delete(Config) ->
+    Port = ?config(port, Config),
+    %% assert we there's no bridges at first
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+
+    %% then we add a webhook bridge, using POST
+    %% POST /bridges/ will create a bridge
+    URL1 = ?URL(Port, "path1"),
+    Name = <<"t_http_crud_apis">>,
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    {ok, 201, _} = request(
+        post,
+        uri(["bridges"]),
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+    ),
+    {ok, 201, Rule} = request(
+        post,
+        uri(["rules"]),
+        #{
+            <<"name">> => <<"t_http_crud_apis">>,
+            <<"enable">> => true,
+            <<"actions">> => [BridgeID],
+            <<"sql">> => <<"SELECT * from \"t\"">>
+        }
+    ),
+    #{<<"id">> := RuleId} = jsx:decode(Rule),
+    %% delete the bridge should fail because there is a rule depenents on it
+    {ok, 403, _} = request(delete, uri(["bridges", BridgeID]), []),
+    %% delete the rule first
+    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
+    %% then delete the bridge is OK
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+    ok.
+
+t_cascade_delete_actions(Config) ->
+    Port = ?config(port, Config),
+    %% assert we there's no bridges at first
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+
+    %% then we add a webhook bridge, using POST
+    %% POST /bridges/ will create a bridge
+    URL1 = ?URL(Port, "path1"),
+    Name = <<"t_http_crud_apis">>,
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    {ok, 201, _} = request(
+        post,
+        uri(["bridges"]),
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+    ),
+    {ok, 201, Rule} = request(
+        post,
+        uri(["rules"]),
+        #{
+            <<"name">> => <<"t_http_crud_apis">>,
+            <<"enable">> => true,
+            <<"actions">> => [BridgeID],
+            <<"sql">> => <<"SELECT * from \"t\"">>
+        }
+    ),
+    #{<<"id">> := RuleId} = jsx:decode(Rule),
+    %% delete the bridge will also delete the actions from the rules
+    {ok, 204, _} = request(delete, uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions", []),
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+    {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
+    ?assertMatch(
+        #{
+            <<"actions">> := []
+        },
+        jsx:decode(Rule1)
+    ),
+    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
+    ok.
+
 t_start_stop_bridges_node(Config) ->
     do_start_stop_bridges(node, Config).
 
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl
index 6b3891ef3..b74a118d2 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl
@@ -92,7 +92,7 @@ request(Username, Method, Url, Body) ->
 uri() -> uri([]).
 uri(Parts) when is_list(Parts) ->
     NParts = [E || E <- Parts],
-    ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]).
+    ?HOST ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])).
 
 auth_header(Username) ->
     Password = <<"public">>,
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
index f991c7b4f..5995b3f53 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
@@ -46,6 +46,8 @@
     get_rules/0,
     get_rules_for_topic/1,
     get_rules_with_same_event/1,
+    get_rule_ids_by_action/1,
+    ensure_action_removed/2,
     get_rules_ordered_by_ts/0
 ]).
 
@@ -99,6 +101,8 @@
 
 -define(RATE_METRICS, ['matched']).
 
+-type action_name() :: binary() | #{function := binary()}.
+
 config_key_path() ->
     [rule_engine, rules].
 
@@ -208,6 +212,46 @@ get_rules_with_same_event(Topic) ->
         lists:any(fun(T) -> is_of_event_name(EventName, T) end, From)
     ].
 
+-spec get_rule_ids_by_action(action_name()) -> [rule_id()].
+get_rule_ids_by_action(ActionName) when is_binary(ActionName) ->
+    [
+        Id
+     || #{actions := Acts, id := Id} <- get_rules(),
+        lists:any(fun(A) -> A =:= ActionName end, Acts)
+    ];
+get_rule_ids_by_action(#{function := FuncName}) when is_binary(FuncName) ->
+    {Mod, Fun} =
+        case string:split(FuncName, ":", leading) of
+            [M, F] -> {binary_to_module(M), F};
+            [F] -> {emqx_rule_actions, F}
+        end,
+    [
+        Id
+     || #{actions := Acts, id := Id} <- get_rules(),
+        contains_actions(Acts, Mod, Fun)
+    ].
+
+-spec ensure_action_removed(rule_id(), action_name()) -> ok.
+ensure_action_removed(RuleId, ActionName) ->
+    FilterFunc =
+        fun
+            (Func, Func) -> false;
+            (#{<<"function">> := Func}, #{function := Func}) -> false;
+            (_, _) -> true
+        end,
+    case emqx:get_raw_config([rule_engine, rules, RuleId], not_found) of
+        not_found ->
+            ok;
+        #{<<"actions">> := Acts} ->
+            NewActs = [AName || AName <- Acts, FilterFunc(AName, ActionName)],
+            {ok, _} = emqx_conf:update(
+                emqx_rule_engine:config_key_path() ++ [RuleId, actions],
+                NewActs,
+                #{override_to => cluster}
+            ),
+            ok
+    end.
+
 is_of_event_name(EventName, Topic) ->
     EventName =:= emqx_rule_events:event_name(Topic).
 
@@ -413,3 +457,20 @@ now_ms() ->
 
 bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
 bin(B) when is_binary(B) -> B.
+
+binary_to_module(ModName) ->
+    try
+        binary_to_existing_atom(ModName, utf8)
+    catch
+        error:badarg ->
+            not_exist_mod
+    end.
+
+contains_actions(Actions, Mod0, Func0) ->
+    lists:any(
+        fun
+            (#{mod := Mod, func := Func}) when Mod =:= Mod0; Func =:= Func0 -> true;
+            (_) -> false
+        end,
+        Actions
+    ).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
index 9669e0113..4de63e94f 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
@@ -19,7 +19,6 @@
 
 -export([
     test/1,
-    echo_action/2,
     get_selected_data/3
 ]).
 
@@ -70,7 +69,8 @@ test_rule(Sql, Select, Context, EventTopics) ->
         ok = emqx_rule_engine:clear_metrics_for_rule(RuleId)
     end.
 
-get_selected_data(Selected, _Envs, _Args) ->
+get_selected_data(Selected, Envs, Args) ->
+    ?TRACE("RULE", "testing_rule_sql_ok", #{selected => Selected, envs => Envs, args => Args}),
     {ok, Selected}.
 
 is_publish_topic(<<"$events/", _/binary>>) -> false;
@@ -84,10 +84,6 @@ flatten([{ok, D}]) ->
 flatten([D | L]) when is_list(D) ->
     [D0 || {ok, D0} <- D] ++ flatten(L).
 
-echo_action(Data, Envs) ->
-    ?TRACE("RULE", "testing_rule_sql_ok", #{data => Data, envs => Envs}),
-    {ok, Data}.
-
 fill_default_values(Event, Context) ->
     maps:merge(envs_examp(Event), Context).
 
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index 4e0c7dfe3..ba3a2cc30 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -52,7 +52,9 @@ groups() ->
             t_create_existing_rule,
             t_get_rules_for_topic,
             t_get_rules_for_topic_2,
-            t_get_rules_with_same_event
+            t_get_rules_with_same_event,
+            t_get_rule_ids_by_action,
+            t_ensure_action_removed
         ]},
         {runtime, [], [
             t_match_atom_and_binary,
@@ -431,6 +433,105 @@ t_get_rules_with_same_event(_Config) ->
     ]),
     ok.
 
+t_get_rule_ids_by_action(_) ->
+    ID = <<"t_get_rule_ids_by_action">>,
+    Rule1 = #{
+        enable => false,
+        id => ID,
+        sql => <<"SELECT * FROM \"t\"">>,
+        from => [<<"t">>],
+        fields => [<<"*">>],
+        is_foreach => false,
+        conditions => {},
+        actions => [
+            #{mod => emqx_rule_actions, func => console, args => #{}},
+            #{mod => emqx_rule_actions, func => republish, args => #{}},
+            <<"mqtt:my_mqtt_bridge">>,
+            <<"mysql:foo">>
+        ],
+        description => ID,
+        created_at => erlang:system_time(millisecond)
+    },
+    ok = insert_rules([Rule1]),
+    ?assertMatch(
+        [ID],
+        emqx_rule_engine:get_rule_ids_by_action(#{function => <<"emqx_rule_actions:console">>})
+    ),
+    ?assertMatch(
+        [ID],
+        emqx_rule_engine:get_rule_ids_by_action(#{function => <<"emqx_rule_actions:republish">>})
+    ),
+    ?assertEqual([], emqx_rule_engine:get_rule_ids_by_action(#{function => <<"some_mod:fun">>})),
+    ?assertMatch([ID], emqx_rule_engine:get_rule_ids_by_action(<<"mysql:foo">>)),
+    ?assertEqual([], emqx_rule_engine:get_rule_ids_by_action(<<"mysql:not_exists">>)),
+    ok = delete_rules_by_ids([<<"t_get_rule_ids_by_action">>]).
+
+t_ensure_action_removed(_) ->
+    Id = <<"t_ensure_action_removed">>,
+    GetSelectedData = <<"emqx_rule_sqltester:get_selected_data">>,
+    emqx:update_config(
+        [rule_engine, rules],
+        #{
+            Id => #{
+                <<"actions">> => [
+                    #{<<"function">> => GetSelectedData},
+                    #{<<"function">> => <<"console">>},
+                    #{<<"function">> => <<"republish">>},
+                    <<"mysql:foo">>,
+                    <<"mqtt:bar">>
+                ],
+                <<"description">> => <<"">>,
+                <<"sql">> => <<"SELECT * FROM \"t/#\"">>
+            }
+        }
+    ),
+    ?assertMatch(
+        #{
+            <<"actions">> := [
+                #{<<"function">> := GetSelectedData},
+                #{<<"function">> := <<"console">>},
+                #{<<"function">> := <<"republish">>},
+                <<"mysql:foo">>,
+                <<"mqtt:bar">>
+            ]
+        },
+        emqx:get_raw_config([rule_engine, rules, Id])
+    ),
+    ok = emqx_rule_engine:ensure_action_removed(Id, #{function => <<"console">>}),
+    ?assertMatch(
+        #{
+            <<"actions">> := [
+                #{<<"function">> := GetSelectedData},
+                #{<<"function">> := <<"republish">>},
+                <<"mysql:foo">>,
+                <<"mqtt:bar">>
+            ]
+        },
+        emqx:get_raw_config([rule_engine, rules, Id])
+    ),
+    ok = emqx_rule_engine:ensure_action_removed(Id, <<"mysql:foo">>),
+    ?assertMatch(
+        #{
+            <<"actions">> := [
+                #{<<"function">> := GetSelectedData},
+                #{<<"function">> := <<"republish">>},
+                <<"mqtt:bar">>
+            ]
+        },
+        emqx:get_raw_config([rule_engine, rules, Id])
+    ),
+    ok = emqx_rule_engine:ensure_action_removed(Id, #{function => GetSelectedData}),
+    ?assertMatch(
+        #{
+            <<"actions">> := [
+                #{<<"function">> := <<"republish">>},
+                <<"mqtt:bar">>
+            ]
+        },
+        emqx:get_raw_config([rule_engine, rules, Id])
+    ),
+    emqx:remove_config([rule_engine, rules, Id]).
+
 %%------------------------------------------------------------------------------
 %% Test cases for rule runtime
 %%------------------------------------------------------------------------------

From 499da1ebe02d0ae6263a6eef5d2d6989cda33e7d Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Tue, 6 Sep 2022 15:46:18 +0800
Subject: [PATCH 134/232] chore: add emqx_release:edition_longstr()

---
 apps/emqx/src/emqx_release.erl                          | 5 +++++
 apps/emqx_management/src/emqx_mgmt.erl                  | 9 +--------
 apps/emqx_management/src/emqx_mgmt_api_nodes.erl        | 4 ++--
 apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl | 5 +----
 4 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/apps/emqx/src/emqx_release.erl b/apps/emqx/src/emqx_release.erl
index 62dcd89dc..f6a3db5d0 100644
--- a/apps/emqx/src/emqx_release.erl
+++ b/apps/emqx/src/emqx_release.erl
@@ -18,6 +18,7 @@
 
 -export([
     edition/0,
+    edition_longstr/0,
     description/0,
     version/0
 ]).
@@ -44,8 +45,12 @@ description() ->
 -spec edition() -> ce | ee.
 -ifdef(EMQX_RELEASE_EDITION).
 edition() -> ?EMQX_RELEASE_EDITION.
+
+edition_longstr() -> <<"Enterprise">>.
 -else.
 edition() -> ce.
+
+edition_longstr() -> <<"Opensource">>.
 -endif.
 
 %% @doc Return the release version.
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index eefee0dfc..d1232c122 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -19,7 +19,6 @@
 -include("emqx_mgmt.hrl").
 -elvis([{elvis_style, invalid_dynamic_call, disable}]).
 -elvis([{elvis_style, god_modules, disable}]).
--dialyzer({nowarn_function, edition/0}).
 
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("emqx/include/emqx.hrl").
@@ -142,7 +141,7 @@ node_info() ->
         node_status => 'running',
         uptime => proplists:get_value(uptime, BrokerInfo),
         version => iolist_to_binary(proplists:get_value(version, BrokerInfo)),
-        edition => edition(),
+        edition => emqx_release:edition_longstr(),
         role => mria_rlog:role()
     }.
 
@@ -555,9 +554,3 @@ max_row_limit() ->
     ?MAX_ROW_LIMIT.
 
 table_size(Tab) -> ets:info(Tab, size).
-
-edition() ->
-    case emqx_release:edition() of
-        ee -> <<"enterprise">>;
-        ce -> <<"opensource">>
-    end.
diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
index c58166b1a..d0d2e4b8c 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl
@@ -219,8 +219,8 @@ fields(node_info) ->
             )},
         {edition,
             mk(
-                enum([opensource, enterprise]),
-                #{desc => <<"Release edition">>, example => "opensource"}
+                enum(['Opensource', 'Enterprise']),
+                #{desc => <<"Release edition">>, example => "Opensource"}
             )},
         {sys_path,
             mk(
diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
index d031b3d09..c2330dc48 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
@@ -58,10 +58,7 @@ t_nodes_api(_) ->
     Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8),
     ?assertEqual(Node, node()),
     Edition = maps:get(<<"edition">>, LocalNodeInfo),
-    case emqx_release:edition() of
-        ee -> ?assertEqual(<<"enterprise">>, Edition);
-        ce -> ?assertEqual(<<"opensource">>, Edition)
-    end,
+    ?assertEqual(emqx_release:edition_longstr(), Edition),
 
     NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]),
     {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath),

From 2b33ca6d4924162eeb8f4ae322caf44fd25d60bd Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 7 Sep 2022 16:00:09 +0800
Subject: [PATCH 135/232] fix: no error log print if insert bool values into
 mysql

---
 .../src/emqx_connector_mysql.erl              | 16 +++++---
 .../include/emqx_resource_errors.hrl          |  2 +-
 .../src/emqx_resource_worker.erl              | 39 ++++++++++++-------
 3 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index 802427134..8b44846cf 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -398,9 +398,8 @@ on_sql_query(
     ?TRACE("QUERY", "mysql_connector_received", LogMeta),
     Worker = ecpool:get_client(PoolName),
     {ok, Conn} = ecpool_worker:client(Worker),
-    Result = erlang:apply(mysql, SQLFunc, [Conn, SQLOrKey, Data, Timeout]),
-    case Result of
-        {error, disconnected} ->
+    try mysql:SQLFunc(Conn, SQLOrKey, Data, Timeout) of
+        {error, disconnected} = Result ->
             ?SLOG(
                 error,
                 LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected}
@@ -421,12 +420,19 @@ on_sql_query(
                 LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
             ),
             {error, {recoverable_error, Reason}};
-        {error, Reason} ->
+        {error, Reason} = Result ->
             ?SLOG(
                 error,
                 LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
             ),
             Result;
-        _ ->
+        Result ->
             Result
+    catch
+        error:badarg ->
+            ?SLOG(
+                error,
+                LogMeta#{msg => "mysql_connector_invalid_params", params => Data}
+            ),
+            {error, {invalid_params, Data}}
     end.
diff --git a/apps/emqx_resource/include/emqx_resource_errors.hrl b/apps/emqx_resource/include/emqx_resource_errors.hrl
index b11ee3c1a..6d1b3e92f 100644
--- a/apps/emqx_resource/include/emqx_resource_errors.hrl
+++ b/apps/emqx_resource/include/emqx_resource_errors.hrl
@@ -15,6 +15,6 @@
 %%--------------------------------------------------------------------
 
 -define(RESOURCE_ERROR(Reason, Msg),
-    {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}}
+    {error, {resource_error, #{reason => Reason, msg => Msg}}}
 ).
 -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}).
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index c5841b7ef..a451939b6 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -374,18 +374,21 @@ reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
     gen_statem:reply(From, Result),
     handle_query_result(Id, Result, BlockWorker).
 
-handle_query_result(Id, ?RESOURCE_ERROR_M(exception, _), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(exception, Msg), BlockWorker) ->
+    ?SLOG(error, #{msg => resource_exception, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
 handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _) when
     NotWorking == not_connected; NotWorking == blocked
 ->
     true;
-handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, _), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), BlockWorker) ->
+    ?SLOG(error, #{msg => resource_not_found, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_not_found'),
     BlockWorker;
-handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, _), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), BlockWorker) ->
+    ?SLOG(error, #{msg => resource_stopped, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_stopped'),
     BlockWorker;
@@ -394,18 +397,21 @@ handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.other'),
     BlockWorker;
-handle_query_result(Id, {error, {recoverable_error, _}}, _BlockWorker) ->
+handle_query_result(Id, {error, {recoverable_error, Reason}}, _BlockWorker) ->
     %% the message will be queued in replayq or inflight window,
     %% i.e. the counter 'queuing' will increase, so we pretend that we have not
     %% sent this message.
+    ?SLOG(warning, #{msg => recoverable_error, reason => Reason}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', -1),
     true;
-handle_query_result(Id, {error, _}, BlockWorker) ->
+handle_query_result(Id, {error, Reason}, BlockWorker) ->
+    ?SLOG(error, #{msg => send_error, reason => Reason}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
 handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
     true;
-handle_query_result(Id, {async_return, {error, _}}, BlockWorker) ->
+handle_query_result(Id, {async_return, {error, Msg}}, BlockWorker) ->
+    ?SLOG(error, #{msg => async_send_error, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
     BlockWorker;
 handle_query_result(_Id, {async_return, ok}, BlockWorker) ->
@@ -433,7 +439,7 @@ call_query(QM0, Id, Query, QueryOpts) ->
             ?RESOURCE_ERROR(not_found, "resource not found")
     end.
 
--define(APPLY_RESOURCE(EXPR, REQ),
+-define(APPLY_RESOURCE(NAME, EXPR, REQ),
     try
         %% if the callback module (connector) wants to return an error that
         %% makes the current resource goes into the `blocked` state, it should
@@ -441,12 +447,13 @@ call_query(QM0, Id, Query, QueryOpts) ->
         EXPR
     catch
         ERR:REASON:STACKTRACE ->
-            MSG = io_lib:format(
-                "call query failed, func: ~s, id: ~s, error: ~0p, Request: ~0p",
-                [??EXPR, Id, {ERR, REASON, STACKTRACE}, REQ],
-                [{chars_limit, 1024}]
-            ),
-            ?RESOURCE_ERROR(exception, MSG)
+            ?RESOURCE_ERROR(exception, #{
+                name => NAME,
+                id => Id,
+                request => REQ,
+                error => {ERR, REASON},
+                stacktrace => STACKTRACE
+            })
     end
 ).
 
@@ -454,7 +461,7 @@ apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) -
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
-    Result = ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request),
+    Result = ?APPLY_RESOURCE(call_query, Mod:on_query(Id, Request, ResSt), Request),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
     Result;
 apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
@@ -462,6 +469,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     Name = maps:get(inflight_name, QueryOpts, undefined),
     WinSize = maps:get(inflight_window, QueryOpts, undefined),
     ?APPLY_RESOURCE(
+        call_query_async,
         case inflight_is_full(Name, WinSize) of
             true ->
                 ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),
@@ -484,7 +492,7 @@ apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
     BatchLen = length(Batch),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', BatchLen),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', BatchLen),
-    Result = ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch),
+    Result = ?APPLY_RESOURCE(call_batch_query, Mod:on_batch_query(Id, Requests, ResSt), Batch),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -BatchLen),
     Result;
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
@@ -492,6 +500,7 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     Name = maps:get(inflight_name, QueryOpts, undefined),
     WinSize = maps:get(inflight_window, QueryOpts, undefined),
     ?APPLY_RESOURCE(
+        call_batch_query_async,
         case inflight_is_full(Name, WinSize) of
             true ->
                 ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),

From 7898867c6857c05c0702bdb37493db760968db95 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 8 Sep 2022 13:13:10 -0300
Subject: [PATCH 136/232] chore(mix): guard enterprise-only dependencies

Avoid downloading enterprise-only dependencies in community edition.
---
 Makefile | 24 +++---------------------
 build    |  6 ++++++
 mix.exs  | 19 +++++++++++++++----
 3 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index 85cb3241a..f98449081 100644
--- a/Makefile
+++ b/Makefile
@@ -44,22 +44,6 @@ all: $(REBAR) $(PROFILES)
 ensure-rebar3:
 	@$(SCRIPTS)/ensure-rebar3.sh
 
-.PHONY: ensure-hex
-ensure-hex:
-	@mix local.hex --if-missing --force
-
-.PHONY: ensure-mix-rebar3
-ensure-mix-rebar3: $(REBAR)
-	@mix local.rebar rebar3 $(CURDIR)/rebar3 --if-missing --force
-
-.PHONY: ensure-mix-rebar
-ensure-mix-rebar: $(REBAR)
-	@mix local.rebar --if-missing --force
-
-.PHONY: mix-deps-get
-mix-deps-get: $(ELIXIR_COMMON_DEPS)
-	@mix deps.get
-
 $(REBAR): prepare ensure-rebar3
 
 .PHONY: eunit
@@ -116,8 +100,6 @@ coveralls: $(REBAR)
 
 COMMON_DEPS := $(REBAR)
 
-ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
-
 .PHONY: $(REL_PROFILES)
 $(REL_PROFILES:%=%): $(COMMON_DEPS)
 	@$(BUILD) $(@) rel
@@ -226,13 +208,13 @@ conf-segs:
 
 ## elixir target is to create release packages using Elixir's Mix
 .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir)
-$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
+$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS)
 	@env IS_ELIXIR=yes $(BUILD) $(subst -elixir,,$(@)) elixir
 
 .PHONY: $(REL_PROFILES:%=%-elixir-pkg)
 define gen-elixir-pkg-target
 # the Elixir places the tar in a different path than Rebar3
-$1-elixir-pkg: $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
+$1-elixir-pkg: $(COMMON_DEPS)
 	@env TAR_PKG_DIR=_build/$1-pkg \
 	     IS_ELIXIR=yes \
 	     $(BUILD) $1-pkg pkg
@@ -241,7 +223,7 @@ $(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt))))
 
 .PHONY: $(REL_PROFILES:%=%-elixir-tgz)
 define gen-elixir-tgz-target
-$1-elixir-tgz: $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
+$1-elixir-tgz: $(COMMON_DEPS)
 	@env IS_ELIXIR=yes $(BUILD) $1 tgz
 endef
 ALL_ELIXIR_TGZS = $(REL_PROFILES)
diff --git a/build b/build
index 5f0c96744..01efc3388 100755
--- a/build
+++ b/build
@@ -147,6 +147,12 @@ make_rel() {
 make_elixir_rel() {
   ./scripts/pre-compile.sh "$PROFILE"
   export_release_vars "$PROFILE"
+  # for some reason, this has to be run outside "do"...
+  mix local.rebar --if-missing --force
+  # shellcheck disable=SC1010
+  mix do local.hex --if-missing --force, \
+      local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
+      deps.get
   mix release --overwrite
   assert_no_compile_time_only_deps
 }
diff --git a/mix.exs b/mix.exs
index 3858b6532..b8a977955 100644
--- a/mix.exs
+++ b/mix.exs
@@ -88,11 +88,11 @@ defmodule EMQXUmbrella.MixProject do
       {:ranch,
        github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
       # in conflict by grpc and eetcd
-      {:gpb, "4.11.2", override: true, runtime: false},
-      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
-      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.3", override: true}
+      {:gpb, "4.11.2", override: true, runtime: false}
     ] ++
-      umbrella_apps() ++ enterprise_apps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
+      umbrella_apps() ++
+      enterprise_apps(profile_info) ++
+      enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
   end
 
   defp umbrella_apps() do
@@ -126,6 +126,17 @@ defmodule EMQXUmbrella.MixProject do
     []
   end
 
+  defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
+    [
+      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
+      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.3", override: true}
+    ]
+  end
+
+  defp enterprise_deps(_profile_info) do
+    []
+  end
+
   defp releases() do
     [
       emqx: fn ->

From b9ae4ea27694af87f5ea1664dbc565a82fd998d8 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 13 Sep 2022 13:56:53 +0800
Subject: [PATCH 137/232] refactor: rename some metrics for emqx_resource

---
 apps/emqx_bridge/i18n/emqx_bridge_schema.conf | 10 ++--
 apps/emqx_bridge/include/emqx_bridge.hrl      | 20 +++----
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 10 ++--
 .../src/schema/emqx_bridge_schema.erl         |  8 +--
 .../test/emqx_bridge_mqtt_SUITE.erl           |  8 +--
 .../src/emqx_resource_manager.erl             | 12 ++--
 .../src/emqx_resource_worker.erl              | 57 ++++++++-----------
 .../test/emqx_resource_SUITE.erl              |  8 +--
 8 files changed, 62 insertions(+), 71 deletions(-)

diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
index de9e10e74..08fe9c299 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
@@ -171,14 +171,14 @@ emqx_bridge_schema {
                            zh: "被缓存"
                           }
                   }
-     metric_sent {
+     metric_retried {
                    desc {
-                         en: """Count of messages that are sent by this bridge."""
-                         zh: """已经发送出去的消息个数。"""
+                         en: """Times of retried from the queue or the inflight window."""
+                         zh: """从队列或者飞行窗口里重试的次数。"""
                         }
                    label: {
-                           en: "Sent"
-                           zh: "已发送"
+                           en: "Retried"
+                           zh: "已重试"
                           }
                   }
 
diff --git a/apps/emqx_bridge/include/emqx_bridge.hrl b/apps/emqx_bridge/include/emqx_bridge.hrl
index 2b64dba70..6bc80f9cc 100644
--- a/apps/emqx_bridge/include/emqx_bridge.hrl
+++ b/apps/emqx_bridge/include/emqx_bridge.hrl
@@ -14,7 +14,7 @@
     DroppedResourceStopped,
     Matched,
     Queued,
-    Sent,
+    Retried,
     SentFailed,
     SentInflight,
     SentSucc,
@@ -33,10 +33,10 @@
         'dropped.resource_stopped' => DroppedResourceStopped,
         'matched' => Matched,
         'queuing' => Queued,
-        'sent' => Sent,
-        'sent.failed' => SentFailed,
-        'sent.inflight' => SentInflight,
-        'sent.success' => SentSucc,
+        'retried' => Retried,
+        'failed' => SentFailed,
+        'inflight' => SentInflight,
+        'success' => SentSucc,
         rate => RATE,
         rate_last5m => RATE_5,
         rate_max => RATE_MAX,
@@ -54,7 +54,7 @@
     DroppedResourceStopped,
     Matched,
     Queued,
-    Sent,
+    Retried,
     SentFailed,
     SentInflight,
     SentSucc,
@@ -73,10 +73,10 @@
         'dropped.resource_stopped' := DroppedResourceStopped,
         'matched' := Matched,
         'queuing' := Queued,
-        'sent' := Sent,
-        'sent.failed' := SentFailed,
-        'sent.inflight' := SentInflight,
-        'sent.success' := SentSucc,
+        'retried' := Retried,
+        'failed' := SentFailed,
+        'inflight' := SentInflight,
+        'success' := SentSucc,
         rate := RATE,
         rate_last5m := RATE_5,
         rate_max := RATE_MAX,
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index ba896d9b7..a353c9cf0 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -704,10 +704,10 @@ format_metrics(#{
         'dropped.resource_stopped' := DroppedResourceStopped,
         'matched' := Matched,
         'queuing' := Queued,
-        'sent' := Sent,
-        'sent.failed' := SentFailed,
-        'sent.inflight' := SentInflight,
-        'sent.success' := SentSucc,
+        'retried' := Retried,
+        'failed' := SentFailed,
+        'inflight' := SentInflight,
+        'success' := SentSucc,
         'received' := Rcvd
     },
     rate := #{
@@ -724,7 +724,7 @@ format_metrics(#{
         DroppedResourceStopped,
         Matched,
         Queued,
-        Sent,
+        Retried,
         SentFailed,
         SentInflight,
         SentSucc,
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 3beeb6e28..8bfc1c78a 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -118,10 +118,10 @@ fields("metrics") ->
             mk(integer(), #{desc => ?DESC("metric_dropped_resource_stopped")})},
         {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})},
         {"queuing", mk(integer(), #{desc => ?DESC("metric_queuing")})},
-        {"sent", mk(integer(), #{desc => ?DESC("metric_sent")})},
-        {"sent.failed", mk(integer(), #{desc => ?DESC("metric_sent_failed")})},
-        {"sent.inflight", mk(integer(), #{desc => ?DESC("metric_sent_inflight")})},
-        {"sent.success", mk(integer(), #{desc => ?DESC("metric_sent_success")})},
+        {"retried", mk(integer(), #{desc => ?DESC("metric_retried")})},
+        {"failed", mk(integer(), #{desc => ?DESC("metric_sent_failed")})},
+        {"inflight", mk(integer(), #{desc => ?DESC("metric_sent_inflight")})},
+        {"success", mk(integer(), #{desc => ?DESC("metric_sent_success")})},
         {"rate", mk(float(), #{desc => ?DESC("metric_rate")})},
         {"rate_max", mk(float(), #{desc => ?DESC("metric_rate_max")})},
         {"rate_last5m",
diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
index 02b76d64b..e35ad5fe5 100644
--- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -245,13 +245,13 @@ t_mqtt_conn_bridge_egress(_) ->
     {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
     ?assertMatch(
         #{
-            <<"metrics">> := #{<<"matched">> := 1, <<"sent.success">> := 1, <<"sent.failed">> := 0},
+            <<"metrics">> := #{<<"matched">> := 1, <<"success">> := 1, <<"failed">> := 0},
             <<"node_metrics">> :=
                 [
                     #{
                         <<"node">> := _,
                         <<"metrics">> :=
-                            #{<<"matched">> := 1, <<"sent.success">> := 1, <<"sent.failed">> := 0}
+                            #{<<"matched">> := 1, <<"success">> := 1, <<"failed">> := 0}
                     }
                 ]
         },
@@ -464,13 +464,13 @@ t_egress_mqtt_bridge_with_rules(_) ->
     {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
     ?assertMatch(
         #{
-            <<"metrics">> := #{<<"matched">> := 2, <<"sent.success">> := 2, <<"sent.failed">> := 0},
+            <<"metrics">> := #{<<"matched">> := 2, <<"success">> := 2, <<"failed">> := 0},
             <<"node_metrics">> :=
                 [
                     #{
                         <<"node">> := _,
                         <<"metrics">> := #{
-                            <<"matched">> := 2, <<"sent.success">> := 2, <<"sent.failed">> := 0
+                            <<"matched">> := 2, <<"success">> := 2, <<"failed">> := 0
                         }
                     }
                 ]
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index c8c097c30..e4ba92b5c 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -130,18 +130,18 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         ResId,
         [
             'matched',
-            'sent',
+            'retried',
+            'success',
+            'failed',
             'dropped',
-            'queuing',
-            'batching',
-            'sent.success',
-            'sent.failed',
-            'sent.inflight',
             'dropped.queue_not_enabled',
             'dropped.queue_full',
             'dropped.resource_not_found',
             'dropped.resource_stopped',
             'dropped.other',
+            'queuing',
+            'batching',
+            'inflight',
             'received'
         ],
         [matched]
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index a451939b6..f683b67ed 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -250,6 +250,7 @@ retry_first_from_queue(Q, Id, St) ->
     end.
 
 retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = St0) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
     Result = call_query(sync, Id, FirstQuery, #{}),
     case handle_query_result(Id, Result, false) of
         %% Send failed because resource down
@@ -376,49 +377,48 @@ reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
 
 handle_query_result(Id, ?RESOURCE_ERROR_M(exception, Msg), BlockWorker) ->
     ?SLOG(error, #{msg => resource_exception, info => Msg}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
     BlockWorker;
 handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _) when
     NotWorking == not_connected; NotWorking == blocked
 ->
     true;
 handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), BlockWorker) ->
-    ?SLOG(error, #{msg => resource_not_found, info => Msg}),
+    ?SLOG(error, #{id => Id, msg => resource_not_found, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_not_found'),
     BlockWorker;
 handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), BlockWorker) ->
-    ?SLOG(error, #{msg => resource_stopped, info => Msg}),
+    ?SLOG(error, #{id => Id, msg => resource_stopped, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_stopped'),
     BlockWorker;
 handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
-    ?SLOG(error, #{msg => other_resource_error, reason => Reason}),
+    ?SLOG(error, #{id => Id, msg => other_resource_error, reason => Reason}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.other'),
     BlockWorker;
 handle_query_result(Id, {error, {recoverable_error, Reason}}, _BlockWorker) ->
     %% the message will be queued in replayq or inflight window,
-    %% i.e. the counter 'queuing' will increase, so we pretend that we have not
+    %% i.e. the counter 'queuing' or 'dropped' will increase, so we pretend that we have not
     %% sent this message.
-    ?SLOG(warning, #{msg => recoverable_error, reason => Reason}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', -1),
+    ?SLOG(warning, #{id => Id, msg => recoverable_error, reason => Reason}),
     true;
 handle_query_result(Id, {error, Reason}, BlockWorker) ->
-    ?SLOG(error, #{msg => send_error, reason => Reason}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
+    ?SLOG(error, #{id => Id, msg => send_error, reason => Reason}),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
     BlockWorker;
 handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
     true;
 handle_query_result(Id, {async_return, {error, Msg}}, BlockWorker) ->
-    ?SLOG(error, #{msg => async_send_error, info => Msg}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.failed'),
+    ?SLOG(error, #{id => Id, msg => async_send_error, info => Msg}),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
     BlockWorker;
 handle_query_result(_Id, {async_return, ok}, BlockWorker) ->
     BlockWorker;
 handle_query_result(Id, Result, BlockWorker) ->
     assert_ok_result(Result),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.success'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'success'),
     BlockWorker.
 
 call_query(QM0, Id, Query, QueryOpts) ->
@@ -459,11 +459,7 @@ call_query(QM0, Id, Query, QueryOpts) ->
 
 apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
-    Result = ?APPLY_RESOURCE(call_query, Mod:on_query(Id, Request, ResSt), Request),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
-    Result;
+    ?APPLY_RESOURCE(call_query, Mod:on_query(Id, Request, ResSt), Request);
 apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
@@ -475,8 +471,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
                 ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent'),
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight'),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight'),
                 ReplyFun = fun ?MODULE:reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = [self(), Id, Name, Ref, Query],
@@ -489,12 +484,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
 apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
     ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Requests = [Request || ?QUERY(_From, Request) <- Batch],
-    BatchLen = length(Batch),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', BatchLen),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', BatchLen),
-    Result = ?APPLY_RESOURCE(call_batch_query, Mod:on_batch_query(Id, Requests, ResSt), Batch),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -BatchLen),
-    Result;
+    ?APPLY_RESOURCE(call_batch_query, Mod:on_batch_query(Id, Requests, ResSt), Batch);
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
@@ -507,8 +497,7 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
                 {async_return, inflight_full};
             false ->
                 BatchLen = length(Batch),
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent', BatchLen),
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', BatchLen),
+                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight', BatchLen),
                 ReplyFun = fun ?MODULE:batch_reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = {ReplyFun, [self(), Id, Name, Ref, Batch]},
@@ -521,12 +510,13 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ).
 
 reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request), Result) ->
-    %% NOTE: 'sent.inflight' is message count that sent but no ACK received,
+    %% NOTE: 'inflight' is message count that sent async but no ACK received,
     %%        NOT the message number ququed in the inflight window.
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -1),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight', -1),
     case reply_caller(Id, ?REPLY(From, Request, Result)) of
         true ->
-            %% we marked these messages are 'queuing' although they are in inflight window
+            %% we marked these messages are 'queuing' although they are actually
+            %% keeped in inflight window, not replayq
             emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
             ?MODULE:block(Pid);
         false ->
@@ -534,13 +524,14 @@ reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request), Result) ->
     end.
 
 batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
-    %% NOTE: 'sent.inflight' is message count that sent but no ACK received,
+    %% NOTE: 'inflight' is message count that sent async but no ACK received,
     %%        NOT the message number ququed in the inflight window.
     BatchLen = length(Batch),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'sent.inflight', -BatchLen),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight', -BatchLen),
     case batch_reply_caller(Id, Result, Batch) of
         true ->
-            %% we marked these messages are 'queuing' although they are in inflight window
+            %% we marked these messages are 'queuing' although they are actually
+            %% keeped in inflight window, not replayq
             emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', BatchLen),
             ?MODULE:block(Pid);
         false ->
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 68b4fb6dd..3f42850ad 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -268,7 +268,7 @@ t_query_counter_async_query(_) ->
         end
     ),
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
-    ?assertMatch(#{matched := 1002, 'sent.success' := 1002, 'sent.failed' := 0}, C),
+    ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C),
     ok = emqx_resource:remove_local(?ID).
 
 t_query_counter_async_callback(_) ->
@@ -309,7 +309,7 @@ t_query_counter_async_callback(_) ->
         end
     ),
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
-    ?assertMatch(#{matched := 1002, sent := 1002, 'sent.success' := 1002, 'sent.failed' := 0}, C),
+    ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C),
     ?assertMatch(1000, ets:info(Tab0, size)),
     ?assert(
         lists:all(
@@ -419,8 +419,8 @@ t_query_counter_async_inflight(_) ->
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
     ct:pal("metrics: ~p", [C]),
     ?assertMatch(
-        #{matched := M, sent := St, 'sent.success' := Ss, dropped := D} when
-            St == Ss andalso M == St + D,
+        #{matched := M, success := Ss, dropped := D} when
+            M == Ss + D,
         C
     ),
     ?assert(

From 6a470dc3ac8b9089140cc3597ba4bbfd635ef25f Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Tue, 13 Sep 2022 18:53:09 +0800
Subject: [PATCH 138/232] feat: don't include emqx-entriprise.conf in
 opensource

---
 mix.exs                      |  4 ++--
 rebar.config.erl             |  2 +-
 scripts/merge-config.escript | 13 ++++++++-----
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/mix.exs b/mix.exs
index 3f55de64b..ff33c09eb 100644
--- a/mix.exs
+++ b/mix.exs
@@ -373,9 +373,9 @@ defmodule EMQXUmbrella.MixProject do
 
     if edition_type == :enterprise do
       render_template(
-        "apps/emqx_conf/etc/emqx_enterprise.conf.all",
+        "apps/emqx_conf/etc/emqx-enterprise.conf.all",
         assigns,
-        Path.join(etc, "emqx_enterprise.conf")
+        Path.join(etc, "emqx-enterprise.conf")
       )
     end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index ce1930ed6..02f075f18 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -477,7 +477,7 @@ emqx_etc_overlay_per_edition(ce) ->
     ];
 emqx_etc_overlay_per_edition(ee) ->
     [
-        {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"},
+        {"{{base_dir}}/lib/emqx_conf/etc/emqx-enterprise.conf.all", "etc/emqx-enterprise.conf"},
         {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
     ].
 
diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript
index 67551bfbe..f617dbb70 100755
--- a/scripts/merge-config.escript
+++ b/scripts/merge-config.escript
@@ -12,22 +12,25 @@
 -define(APPS, ["emqx", "emqx_dashboard", "emqx_authz"]).
 
 main(_) ->
+    Profile = os:getenv("PROFILE", "emqx"),
     {ok, BaseConf} = file:read_file("apps/emqx_conf/etc/emqx_conf.conf"),
-
     Cfgs = get_all_cfgs("apps/"),
+    Enterprise =
+        case Profile of
+            "emqx" -> [];
+            "emqx-enterprise" -> [io_lib:nl(), "include emqx-enterprise.conf", io_lib:nl()]
+        end,
     Conf = [
         merge(BaseConf, Cfgs),
         io_lib:nl(),
-        io_lib:nl(),
-        "include emqx_enterprise.conf",
-        io_lib:nl()
+        Enterprise
     ],
     ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", Conf),
 
     EnterpriseCfgs = get_all_cfgs("lib-ee/"),
     EnterpriseConf = merge("", EnterpriseCfgs),
 
-    ok = file:write_file("apps/emqx_conf/etc/emqx_enterprise.conf.all", EnterpriseConf).
+    ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf).
 
 merge(BaseConf, Cfgs) ->
     lists:foldl(

From 0c1595be0275b43c3686c0bd6dcc123a8979e473 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 12 Sep 2022 10:51:35 +0200
Subject: [PATCH 139/232] feat: Add Kafka connector

---
 .../docker-compose-kafka.yaml                 |  27 +
 .../docker-compose-python.yaml                |   2 +-
 apps/emqx/test/emqx_common_test_helpers.erl   |   9 +
 apps/emqx_bridge/src/emqx_bridge.erl          |   2 +
 apps/emqx_bridge/src/emqx_bridge_resource.erl |   6 +-
 apps/emqx_resource/src/emqx_resource.erl      |  30 +-
 .../src/emqx_resource_manager.erl             |  18 +-
 lib-ee/emqx_ee_bridge/docker-ct               |   1 +
 .../i18n/emqx_ee_bridge_kafka.conf            | 471 ++++++++++++++++++
 lib-ee/emqx_ee_bridge/rebar.config            |   4 +
 .../emqx_ee_bridge/src/emqx_ee_bridge.app.src |   3 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  18 +-
 .../src/emqx_ee_bridge_kafka.erl              | 260 ++++++++++
 .../src/kafka/emqx_bridge_impl_kafka.erl      |  33 ++
 .../kafka/emqx_bridge_impl_kafka_producer.erl | 270 ++++++++++
 .../emqx_bridge_impl_kafka_producer_SUITE.erl |  90 ++++
 .../src/emqx_ee_connector.app.src             |   4 +-
 mix.exs                                       |  12 +-
 rebar.config                                  |   4 +-
 scripts/ct/run.sh                             |  59 ++-
 scripts/find-suites.sh                        |   8 +-
 21 files changed, 1297 insertions(+), 34 deletions(-)
 create mode 100644 .ci/docker-compose-file/docker-compose-kafka.yaml
 create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
 create mode 100644 lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl
 create mode 100644 lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
 create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
new file mode 100644
index 000000000..85532725d
--- /dev/null
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -0,0 +1,27 @@
+version: '3.9'
+
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+    container_name: zookeeper
+    hostname: zookeeper
+    networks:
+      emqx_bridge:
+  kafka_1:
+    image: wurstmeister/kafka:2.13-2.7.0
+    ports:
+      - "9092:9092"
+    container_name: kafka-1.emqx.net
+    hostname: kafka-1.emqx.net
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
+    networks:
+      emqx_bridge:
diff --git a/.ci/docker-compose-file/docker-compose-python.yaml b/.ci/docker-compose-file/docker-compose-python.yaml
index 0b9af4517..14e798c6b 100644
--- a/.ci/docker-compose-file/docker-compose-python.yaml
+++ b/.ci/docker-compose-file/docker-compose-python.yaml
@@ -2,7 +2,7 @@ version: '3.9'
 
 services:
   python:
-    container_name: python 
+    container_name: python
     image: python:3.7.2-alpine3.9
     depends_on:
       - emqx1
diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index ce998656a..24477b21b 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -32,6 +32,7 @@
     stop_apps/1,
     reload/2,
     app_path/2,
+    proj_root/0,
     deps_path/2,
     flush/0,
     flush/1
@@ -245,6 +246,14 @@ stop_apps(Apps) ->
     [application:stop(App) || App <- Apps ++ [emqx, ekka, mria, mnesia]],
     ok.
 
+proj_root() ->
+    filename:join(
+        lists:takewhile(
+            fun(X) -> iolist_to_binary(X) =/= <<"_build">> end,
+            filename:split(app_path(emqx, "."))
+        )
+    ).
+
 %% backward compatible
 deps_path(App, RelativePath) -> app_path(App, RelativePath).
 
diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index d4d24ef3a..ceca5ea7f 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -334,6 +334,8 @@ get_matched_bridges(Topic) ->
         Bridges
     ).
 
+%% TODO: refactor to return bridge type, and bridge name directly
+%% so there is no need to parse the id back to type and name at where it is used
 get_matched_bridge_id(_BType, #{enable := false}, _Topic, _BName, Acc) ->
     Acc;
 get_matched_bridge_id(BType, #{local_topic := Filter}, Topic, BName, Acc) when
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index f0773a8ea..b6fd2e7be 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -44,7 +44,7 @@
 ]).
 
 %% bi-directional bridge with producer/consumer or ingress/egress configs
--define(IS_BI_DIR_BRIDGE(TYPE), TYPE == <<"mqtt">>; TYPE == <<"kafka">>).
+-define(IS_BI_DIR_BRIDGE(TYPE), TYPE =:= <<"mqtt">>; TYPE =:= <<"kafka">>).
 
 -if(?EMQX_RELEASE_EDITION == ee).
 bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt;
@@ -71,6 +71,8 @@ bridge_id(BridgeType, BridgeName) ->
     Type = bin(BridgeType),
     <<Type/binary, ":", Name/binary>>.
 
+parse_bridge_id(<<"bridge:", BridgeId/binary>>) ->
+    parse_bridge_id(BridgeId);
 parse_bridge_id(BridgeId) ->
     case string:split(bin(BridgeId), ":", all) of
         [Type, Name] -> {binary_to_atom(Type, utf8), binary_to_atom(Name, utf8)};
@@ -261,7 +263,7 @@ parse_confs(Type, Name, Conf) when ?IS_BI_DIR_BRIDGE(Type) ->
     %% hookpoint. The underlying driver will run `emqx_hooks:run/3` when it
     %% receives a message from the external database.
     BName = bridge_id(Type, Name),
-    Conf#{hookpoint => <<"$bridges/", BName/binary>>};
+    Conf#{hookpoint => <<"$bridges/", BName/binary>>, bridge_name => Name};
 parse_confs(_Type, _Name, Conf) ->
     Conf.
 
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 309c34195..8086dfa25 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -93,7 +93,8 @@
     %% verify if the resource is working normally
     call_health_check/3,
     %% stop the instance
-    call_stop/3
+    call_stop/3,
+    is_buffer_supported/1
 ]).
 
 %% list all the instances, id only.
@@ -117,7 +118,8 @@
     on_batch_query/3,
     on_query_async/4,
     on_batch_query_async/4,
-    on_get_status/2
+    on_get_status/2,
+    is_buffer_supported/0
 ]).
 
 %% when calling emqx_resource:start/1
@@ -155,6 +157,8 @@
     | {resource_status(), resource_state()}
     | {resource_status(), resource_state(), term()}.
 
+-callback is_buffer_supported() -> boolean().
+
 -spec list_types() -> [module()].
 list_types() ->
     discover_resource_mods().
@@ -256,10 +260,15 @@ query(ResId, Request) ->
     Result :: term().
 query(ResId, Request, Opts) ->
     case emqx_resource_manager:ets_lookup(ResId) of
-        {ok, _Group, #{query_mode := QM}} ->
-            case QM of
-                sync -> emqx_resource_worker:sync_query(ResId, Request, Opts);
-                async -> emqx_resource_worker:async_query(ResId, Request, Opts)
+        {ok, _Group, #{query_mode := QM, mod := Module}} ->
+            IsBufferSupported = is_buffer_supported(Module),
+            case {IsBufferSupported, QM} of
+                {true, _} ->
+                    emqx_resource_worker:simple_sync_query(ResId, Request);
+                {false, sync} ->
+                    emqx_resource_worker:sync_query(ResId, Request, Opts);
+                {false, async} ->
+                    emqx_resource_worker:async_query(ResId, Request, Opts)
             end;
         {error, not_found} ->
             ?RESOURCE_ERROR(not_found, "resource not found")
@@ -336,6 +345,15 @@ list_group_instances(Group) -> emqx_resource_manager:list_group(Group).
 get_callback_mode(Mod) ->
     Mod:callback_mode().
 
+-spec is_buffer_supported(module()) -> boolean().
+is_buffer_supported(Module) ->
+    try
+        Module:is_buffer_supported()
+    catch
+        _:_ ->
+            false
+    end.
+
 -spec call_start(manager_id(), module(), resource_config()) ->
     {ok, resource_state()} | {error, Reason :: term()}.
 call_start(MgrId, Mod, Config) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index e4ba92b5c..82b565f6f 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -146,14 +146,20 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         ],
         [matched]
     ),
-    ok = emqx_resource_worker_sup:start_workers(ResId, Opts),
-    case maps:get(start_after_created, Opts, ?START_AFTER_CREATED) of
+    case emqx_resource:is_buffer_supported(ResourceType) of
         true ->
-            wait_for_ready(ResId, maps:get(start_timeout, Opts, ?START_TIMEOUT));
+            %% the resource it self supports
+            %% buffer, so there is no need for resource workers
+            ok;
         false ->
-            ok
-    end,
-    ok.
+            ok = emqx_resource_worker_sup:start_workers(ResId, Opts),
+            case maps:get(start_after_created, Opts, ?START_AFTER_CREATED) of
+                true ->
+                    wait_for_ready(ResId, maps:get(start_timeout, Opts, ?START_TIMEOUT));
+                false ->
+                    ok
+            end
+    end.
 
 %% @doc Called from `emqx_resource` when doing a dry run for creating a resource instance.
 %%
diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct
index f350a379c..a79037903 100644
--- a/lib-ee/emqx_ee_bridge/docker-ct
+++ b/lib-ee/emqx_ee_bridge/docker-ct
@@ -1,2 +1,3 @@
 mongo
 mongo_rs_sharded
+kafka
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
new file mode 100644
index 000000000..a47bf1249
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
@@ -0,0 +1,471 @@
+emqx_ee_bridge_kafka {
+    config_enable {
+        desc {
+            en: "Enable (true) or disable (false) this Kafka bridge."
+            zh: "启用（true）或停用该（false）Kafka 数据桥接。"
+        }
+        label {
+            en: "Enable or Disable"
+            zh: "启用或停用"
+        }
+    }
+    desc_config {
+        desc {
+            en: """Configuration for a Kafka bridge."""
+            zh: """Kafka 桥接配置"""
+        }
+        label {
+            en: "Kafka Bridge Configuration"
+            zh: "Kafka 桥接配置"
+        }
+    }
+    desc_type {
+        desc {
+            en: """The Bridge Type"""
+            zh: """桥接类型"""
+        }
+        label {
+            en: "Bridge Type"
+            zh: "桥接类型"
+        }
+    }
+    desc_name {
+        desc {
+            en: """Bridge name, used as a human-readable description of the bridge."""
+            zh: """桥接名字，可读描述"""
+        }
+        label {
+            en: "Bridge Name"
+            zh: "桥接名字"
+        }
+    }
+    producer_opts {
+        desc {
+            en: "Local MQTT data source and Kafka bridge configs."
+            zh: "本地 MQTT 数据源和 Kafka 桥接的配置。"
+        }
+        label {
+            en: "MQTT to Kafka"
+            zh: "MQTT 到 Kafka"
+        }
+    }
+    producer_mqtt_opts {
+        desc {
+            en: "MQTT data source. Optional when used as a rule-engine action."
+            zh: "需要桥接到 MQTT 源主题。"
+        }
+        label {
+            en: "MQTT Source Topic"
+            zh: "MQTT 源主题"
+        }
+    }
+    mqtt_topic {
+        desc {
+            en: "MQTT topic or topic as data source (bridge input)."
+            zh: "指定 MQTT 主题作为桥接的数据源"
+        }
+        label {
+            en: "Source MQTT Topic"
+            zh: "源 MQTT 主题"
+        }
+    }
+    producer_kafka_opts {
+        desc {
+            en: "Kafka producer configs."
+            zh: "Kafka 生产者参数。"
+        }
+        label {
+            en: "Kafka Producer"
+            zh: "生产者参数"
+        }
+    }
+    bootstrap_hosts {
+        desc {
+            en: "A comma separated list of Kafka <code>host:port</code> endpoints to bootstrap the client."
+            zh: "用逗号分隔的 <code>host:port</code> 主机列表。"
+        }
+        label {
+            en: "Bootstrap Hosts"
+            zh: "主机列表"
+        }
+    }
+    connect_timeout {
+        desc {
+            en: "Maximum wait time for TCP connection establishment (including authentication time if enabled)."
+            zh: "建立 TCP 连接时的最大等待时长（若启用认证，这个等待时长也包含完成认证所需时间）。"
+        }
+        label {
+            en: "Connect Timeout"
+            zh: "连接超时"
+        }
+    }
+    min_metadata_refresh_interval {
+        desc {
+            en: "Minimum time interval the client has to wait before refreshing Kafka broker and topic metadata. "
+                "Setting too small value may add extra load on Kafka."
+            zh: "刷新 Kafka broker 和 Kafka 主题元数据段最短时间间隔。设置太小可能会增加 Kafka 压力。"
+        }
+        label {
+            en: "Min Metadata Refresh Interval"
+            zh: "元数据刷新最小间隔"
+        }
+    }
+    metadata_request_timeout {
+        desc {
+            en: "Maximum wait time when fetching metadata from Kafka."
+            zh: "刷新元数据时最大等待时长。"
+        }
+        label {
+            en: "Metadata Request Timeout"
+            zh: "元数据请求超时"
+        }
+    }
+    authentication {
+        desc {
+            en: "Authentication configs."
+            zh: "认证参数。"
+        }
+        label {
+            en: "Authentication"
+            zh: "认证"
+        }
+    }
+    socket_opts {
+        desc {
+            en: "Extra socket options."
+            zh: "更多 Socket 参数设置。"
+        }
+        label {
+            en: "Socket Options"
+            zh: "Socket 参数"
+        }
+    }
+    auth_sasl_mechanism {
+        desc {
+            en: "SASL authentication mechanism."
+            zh: "SASL 认证方法名称。"
+        }
+        label {
+            en: "Mechanism"
+            zh: "认证方法"
+        }
+    }
+    auth_sasl_username {
+        desc {
+            en: "SASL authentication username."
+            zh: "SASL 认证的用户名。"
+        }
+        label {
+            en: "Username"
+            zh: "用户名"
+        }
+    }
+    auth_sasl_password {
+        desc {
+            en: "SASL authentication password."
+            zh: "SASL 认证的密码。"
+        }
+        label {
+            en: "Password"
+            zh: "密码"
+        }
+    }
+    auth_kerberos_principal {
+        desc {
+            en: "SASL GSSAPI authentication Kerberos principal. "
+                "For example <code>client_name@MY.KERBEROS.REALM.MYDOMAIN.COM</code>, "
+                "NOTE: The realm in use has to be configured in /etc/krb5.conf in EMQX nodes."
+            zh: "SASL GSSAPI 认证方法的 Kerberos principal，"
+                "例如 <code>client_name@MY.KERBEROS.REALM.MYDOMAIN.COM</code>"
+                "注意：这里使用的 realm 需要配置在 EMQX 服务器的 /etc/krb5.conf 中"
+        }
+        label {
+            en: "Kerberos Principal"
+            zh: "Kerberos Principal"
+        }
+    }
+    auth_kerberos_keytab_file {
+        desc {
+            en: "SASL GSSAPI authentication Kerberos keytab file path. "
+                "NOTE: This file has to be placed in EMQX nodes, and the EMQX service runner user requires read permission."
+            zh: "SASL GSSAPI 认证方法的 Kerberos keytab 文件。"
+                "注意：该文件需要上传到 EMQX 服务器中，且运行 EMQX 服务的系统账户需要有读取权限。"
+        }
+        label {
+            en: "Kerberos keytab file"
+            zh: "Kerberos keytab 文件"
+        }
+    }
+    socket_send_buffer {
+        desc {
+            en: "Fine tune the socket send buffer. The default value is tuned for high throughput."
+            zh: "TCP socket 的发送缓存调优。默认值是针对高吞吐量的一个推荐值。"
+        }
+        label {
+            en: "Socket Send Buffer Size"
+            zh: "Socket 发送缓存大小"
+        }
+    }
+    socket_receive_buffer {
+        desc {
+            en: "Fine tune the socket receive buffer. The default value is tuned for high throughput."
+            zh: "TCP socket 的收包缓存调优。默认值是针对高吞吐量的一个推荐值。"
+        }
+        label {
+            en: "Socket Receive Buffer Size"
+            zh: "Socket 收包缓存大小"
+        }
+    }
+    socket_nodelay {
+        desc {
+            en: "When set to 'true', TCP buffer sent as soon as possible. "
+                "Otherwise the OS kernel may buffer small TCP packets for a while (40ms by default)."
+            zh: "设置 ‘true' 让系统内核立即发送。否则当需要发送当内容很少时，可能会有一定延迟（默认 40 毫秒）。"
+        }
+        label {
+            en: "No Delay"
+            zh: "是否延迟发送"
+        }
+    }
+    kafka_topic {
+        desc {
+            en: "Kafka topic name"
+            zh: "Kafka 主题名称"
+        }
+        label {
+            en: "Kafka Topic Name"
+            zh: "Kafka 主题名称"
+        }
+    }
+    kafka_message {
+        desc {
+            en: "Template to render a Kafka message."
+            zh: "用于生成 Kafka 消息的模版。"
+        }
+        label {
+            en: "Kafka Message Template"
+            zh: "Kafka 消息模版"
+        }
+    }
+    kafka_message_key {
+        desc {
+            en: "Template to render Kafka message key. "
+                "If the desired variable for this template is not found in the input data "
+                "<code>NULL</code> is used."
+            zh: "生成 Kafka 消息 Key 的模版。当所需要的输入没有时，会使用 <code>NULL</code>。"
+        }
+        label {
+            en: "Message Key"
+            zh: "消息的 Key"
+        }
+    }
+    kafka_message_value {
+        desc {
+            en: "Template to render Kafka message value. "
+                "If the desired variable for this template is not found in the input data "
+                "<code>NULL</code> is used."
+            zh: "生成 Kafka 消息 Value 的模版。当所需要的输入没有时，会使用 <code>NULL</code>。"
+        }
+        label {
+            en: "Message Value"
+            zh: "消息的 Value"
+        }
+    }
+    kafka_message_timestamp {
+        desc {
+            en: "Which timestamp to use. "
+                "The timestamp is expected to be a millisecond precision Unix epoch "
+                "which can be in string format, e.g. <code>1661326462115</code> or "
+                "<code>'1661326462115'</code>. "
+                "When the desired data field for this template is not found, "
+                "or if the found data is not a valid integer, "
+                "the current system timestamp will be used."
+            zh: "生成 Kafka 消息时间戳的模版。"
+                "该时间必需是一个整型数值（可以是字符串格式）例如 <code>1661326462115</code> "
+                "或 <code>'1661326462115'</code>。"
+                "当所需的输入字段不存在，或不是一个整型时，"
+                "则会使用当前系统时间。"
+        }
+        label {
+            en: "Message Timestamp"
+            zh: "消息的时间戳"
+        }
+    }
+    max_batch_bytes {
+        desc {
+            en: "Maximum bytes to collect in a Kafka message batch. "
+                "Most of the Kafka brokers default to a limit of 1MB batch size. "
+                "EMQX's default value is less than 1MB in order to compensate "
+                "Kafka message encoding overheads (especially when each individual message is very small). "
+                "When a single message is over the limit, it is still sent (as a single element batch)."
+            zh: "最大消息批量字节数。"
+                "大多数 Kafka 环境的默认最低值是 1MB，EMQX 的默认值比 1MB 更小是因为需要"
+                "补偿 Kafka 消息编码索需要的额外字节（尤其是当每条消息都很小的情况下）。"
+                "当单个消息的大小超过该限制时，它仍然会被发送，（相当于该批量中只有单个消息）。"
+        }
+        label {
+            en: "Max Batch Bytes"
+            zh: "最大批量字节数"
+        }
+    }
+    compression {
+        desc {
+            en: "Compression method."
+            zh: "压缩方法。"
+        }
+        label {
+            en: "Compression"
+            zh: "压缩"
+        }
+    }
+    partition_strategy {
+        desc {
+            en: "Partition strategy is to tell the producer how to dispatch messages to Kafka partitions.\n\n"
+                "<code>random</code>: Randomly pick a partition for each message\n"
+                "<code>key_dispatch</code>: Hash Kafka message key to a partition number\n"
+            zh: "设置消息发布时应该如何选择 Kafka 分区。\n\n"
+                "<code>random</code>: 为每个消息随机选择一个分区。\n"
+                "<code>key_dispatch</code>: Hash Kafka message key to a partition number\n"
+        }
+        label {
+            en: "Partition Strategy"
+            zh: "分区选择策略"
+        }
+    }
+    required_acks {
+        desc {
+            en: "Required acknowledgements for Kafka partition leader to wait for its followers "
+                "before it sends back the acknowledgement to EMQX Kafka producer\n\n"
+                "<code>all_isr</code>: Require all in-sync replicas to acknowledge.\n"
+                "<code>leader_only</code>: Require only the partition-leader's acknowledgement.\n"
+                "<code>none</code>: No need for Kafka to acknowledge at all.\n"
+            zh: "设置 Kafka leader 在返回给 EMQX 确认之前需要等待多少个 follower 的确认。\n\n"
+                "<code>all_isr</code>: 需要所有的在线复制者都确认。\n"
+                "<code>leader_only</code>: 仅需要分区 leader 确认。\n"
+                "<code>none</code>: 无需 Kafka 回复任何确认。\n"
+        }
+        label {
+            en: "Required Acks"
+            zh: "Kafka 确认数量"
+        }
+    }
+    partition_count_refresh_interval {
+        desc {
+            en: "The time interval for Kafka producer to discover increased number of partitions.\n"
+                "After the number of partitions is increased in Kafka, EMQX will start taking the \n"
+                "discovered partitions into account when dispatching messages per <code>partition_strategy</code>."
+            zh: "配置 Kafka 刷新分区数量的时间间隔。\n"
+                "EMQX 发现 Kafka 分区数量增加后，会开始按 <code>partition_strategy<code> 配置，把消息发送到新的分区中。"
+        }
+        label {
+            en: "Partition Count Refresh Interval"
+            zh: "分区数量刷新间隔"
+        }
+    }
+    max_inflight {
+        desc {
+            en: "Maximum number of batches allowed for Kafka producer (per-partition) to send before receiving acknowledgement from Kafka. "
+                "Greater value typically means better throughput. However, there can be a risk of message reordering when this "
+                "value is greater than 1."
+            zh: "设置 Kafka 生产者（每个分区一个）在收到 Kafka 的确认前最多发送多少个请求（批量）。"
+                "调大这个值通常可以增加吞吐量，但是，当该值设置大于 1 是存在消息乱序的风险。"
+        }
+        label {
+            en: "Max Inflight"
+            zh: "飞行窗口"
+        }
+    }
+    producer_buffer {
+        desc {
+            en: "Configure producer message buffer.\n\n"
+                "Tell Kafka producer how to buffer messages when EMQX has more messages to send than "
+                "Kafka can keep up, or when Kafka is down.\n\n"
+            zh: "配置消息缓存的相关参数。\n\n"
+                "当 EMQX 需要发送的消息超过 Kafka 处理能力，或者当 Kafka 临时下线时，EMQX 内部会将消息缓存起来。"
+        }
+        label {
+            en: "Message Buffer"
+            zh: "消息缓存"
+        }
+    }
+    buffer_mode {
+        desc {
+            en: "Message buffer mode.\n\n"
+                "<code>memory</code>: Buffer all messages in memory. The messages will be lost in case of EMQX node restart\n"
+                "<code>disc</code>: Buffer all messages on disk. The messages on disk are able to survive EMQX node restart.\n"
+                "<code>hybrid</code>: Buffer message in memory first, when up to certain limit "
+                "(see <code>segment_bytes</code> config for more information), then start offloading "
+                "messages to disk, Like <code>memory</code> mode, the messages will be lost in case of "
+                "EMQX node restart."
+            zh: "消息缓存模式。\n"
+                "<code>memory</code>: 所有的消息都缓存在内存里。如果 EMQX 服务重启，缓存的消息会丢失。\n"
+                "<code>disc</code>: 缓存到磁盘上。EMQX 重启后会继续发送重启前未发送完成的消息。\n"
+                "<code>hybrid</code>: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制"
+                "（配置项 <code>segment_bytes</code> 描述了该限制）后，后续的消息会缓存到磁盘上。"
+                "与 <code>memory</code> 模式一样，如果 EMQX 服务重启，缓存的消息会丢失。"
+        }
+        label {
+            en: "Buffer Mode"
+            zh: "缓存模式"
+        }
+    }
+    buffer_per_partition_limit {
+        desc {
+            en: "Number of bytes allowed to buffer for each Kafka partition. "
+                "When this limit is exceeded, old messages will be dropped in a trade for credits "
+                "for new messages to be buffered."
+            zh: "为每个 Kafka 分区设置的最大缓存字节数。当超过这个上限之后，老的消息会被丢弃，"
+                "为新的消息腾出空间。"
+        }
+        label {
+            en: "Per-partition Buffer Limit"
+            zh: "Kafka 分区缓存上限"
+        }
+    }
+    buffer_segment_bytes {
+        desc {
+            en: "Applicable when buffer mode is set to <code>disk</code> or <code>hybrid</code>.\n"
+                "This value is to specify the size of each on-disk buffer file."
+            zh: "当缓存模式是 <code>disk</code> 或 <code>hybrid</code> 时适用。"
+                "该配置用于指定缓存到磁盘上的文件的大小。"
+        }
+        label {
+            en: "Segment File Bytes"
+            zh: "缓存文件大小"
+        }
+    }
+    buffer_memory_overload_protection {
+        desc {
+            en: "Applicable when buffer mode is set to <code>memory</code> or <code>hybrid</code>.\n"
+                "EMQX will drop old cached messages under high memory pressure. "
+                "The high memory threshold is defined in config <code>sysmon.os.sysmem_high_watermark</code>."
+            zh: "缓存模式是 <code>memory</code> 或 <code>hybrid</code> 时适用。"
+                "当系统处于高内存压力时，从队列中丢弃旧的消息以减缓内存增长。"
+                "内存压力值由配置项 <code>sysmon.os.sysmem_high_watermark</code> 决定。"
+        }
+        label {
+            en: "Memory Overload Protection"
+            zh: "内存过载保护"
+        }
+    }
+    auth_username_password {
+        desc {
+            en: "Username/password based authentication."
+            zh: "基于用户名密码的认证。"
+        }
+        label {
+            en: "Username/password Auth"
+            zh: "用户名密码认证"
+        }
+    }
+    auth_gssapi_kerberos {
+        desc {
+            en: "Use GSSAPI/Kerberos authentication."
+            zh: "使用 GSSAPI/Kerberos 认证。"
+        }
+        label {
+            en: "GSSAPI/Kerberos"
+            zh: "GSSAPI/Kerberos"
+        }
+    }
+}
diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config
index e986d7983..8c79e7274 100644
--- a/lib-ee/emqx_ee_bridge/rebar.config
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@@ -1,5 +1,9 @@
 {erl_opts, [debug_info]}.
 {deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}
+       , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.6.4"}}}
+       , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
+       , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
+       , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.4"}}}
        , {emqx_connector, {path, "../../apps/emqx_connector"}}
        , {emqx_resource, {path, "../../apps/emqx_resource"}}
        , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index a578b7d0d..97c884fe9 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -3,7 +3,8 @@
     {registered, []},
     {applications, [
         kernel,
-        stdlib
+        stdlib,
+        emqx_ee_connector
     ]},
     {env, []},
     {modules, []},
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 840b963cd..cdf0a6439 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -14,6 +14,7 @@
 
 api_schemas(Method) ->
     [
+        ref(emqx_ee_bridge_kafka, Method),
         ref(emqx_ee_bridge_mysql, Method),
         ref(emqx_ee_bridge_mongodb, Method ++ "_rs"),
         ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"),
@@ -26,6 +27,7 @@ api_schemas(Method) ->
 
 schema_modules() ->
     [
+        emqx_ee_bridge_kafka,
         emqx_ee_bridge_hstreamdb,
         emqx_ee_bridge_influxdb,
         emqx_ee_bridge_mongodb,
@@ -45,6 +47,7 @@ examples(Method) ->
     lists:foldl(Fun, #{}, schema_modules()).
 
 resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8));
+resource_type(kafka) -> emqx_bridge_impl_kafka;
 resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb;
 resource_type(mongodb_rs) -> emqx_connector_mongo;
 resource_type(mongodb_sharded) -> emqx_connector_mongo;
@@ -56,6 +59,11 @@ resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb.
 
 fields(bridges) ->
     [
+        {kafka,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_bridge_kafka, "config")),
+                #{desc => <<"EMQX Enterprise Config">>}
+            )},
         {hstreamdb,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")),
@@ -66,8 +74,9 @@ fields(bridges) ->
                 hoconsc:map(name, ref(emqx_ee_bridge_mysql, "config")),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
-    ] ++ fields(mongodb) ++ fields(influxdb);
-fields(mongodb) ->
+    ] ++ mongodb_structs() ++ influxdb_structs().
+
+mongodb_structs() ->
     [
         {Type,
             mk(
@@ -75,8 +84,9 @@ fields(mongodb) ->
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
      || Type <- [mongodb_rs, mongodb_sharded, mongodb_single]
-    ];
-fields(influxdb) ->
+    ].
+
+influxdb_structs() ->
     [
         {Protocol,
             mk(
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
new file mode 100644
index 000000000..080af35d0
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
@@ -0,0 +1,260 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_kafka).
+
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+
+%% allow atoms like scram_sha_256 and scram_sha_512
+%% i.e. the _256 part does not start with a-z
+-elvis([
+    {elvis_style, atom_naming_convention, #{
+        regex => "^([a-z][a-z0-9]*_?)([a-z0-9]*_?)*$",
+        enclosed_atoms => ".*"
+    }}
+]).
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-export([
+    conn_bridge_examples/1
+]).
+
+-export([
+    namespace/0,
+    roots/0,
+    fields/1,
+    desc/1
+]).
+
+%% -------------------------------------------------------------------------------------------------
+%% api
+
+conn_bridge_examples(Method) ->
+    [
+        #{
+            <<"kafka">> => #{
+                summary => <<"Kafka Bridge">>,
+                value => values(Method)
+            }
+        }
+    ].
+
+values(get) ->
+    maps:merge(values(post), ?METRICS_EXAMPLE);
+values(post) ->
+    #{
+        bootstrap_hosts => <<"localhost:9092">>
+    };
+values(put) ->
+    values(post).
+
+%% -------------------------------------------------------------------------------------------------
+%% Hocon Schema Definitions
+
+namespace() -> "bridge_kafka".
+
+roots() -> ["config"].
+
+fields("post") ->
+    [type_field(), name_field() | fields("config")];
+fields("put") ->
+    fields("config");
+fields("get") ->
+    emqx_bridge_schema:metrics_status_fields() ++ fields("post");
+fields("config") ->
+    [
+        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {bootstrap_hosts, mk(binary(), #{required => true, desc => ?DESC(bootstrap_hosts)})},
+        {connect_timeout,
+            mk(emqx_schema:duration_ms(), #{
+                default => "5s",
+                desc => ?DESC(connect_timeout)
+            })},
+        {min_metadata_refresh_interval,
+            mk(
+                emqx_schema:duration_ms(),
+                #{
+                    default => "3s",
+                    desc => ?DESC(min_metadata_refresh_interval)
+                }
+            )},
+        {metadata_request_timeout,
+            mk(emqx_schema:duration_ms(), #{
+                default => "5s",
+                desc => ?DESC(metadata_request_timeout)
+            })},
+        {authentication,
+            mk(hoconsc:union([none, ref(auth_username_password), ref(auth_gssapi_kerberos)]), #{
+                default => none, desc => ?DESC("authentication")
+            })},
+        {producer, mk(hoconsc:union([none, ref(producer_opts)]), #{desc => ?DESC(producer_opts)})},
+        %{consumer, mk(hoconsc:union([none, ref(consumer_opts)]), #{desc => ?DESC(consumer_opts)})},
+        {socket_opts, mk(ref(socket_opts), #{required => false, desc => ?DESC(socket_opts)})}
+    ] ++ emqx_connector_schema_lib:ssl_fields();
+fields(auth_username_password) ->
+    [
+        {mechanism,
+            mk(enum([plain, scram_sha_256, scram_sha_512]), #{
+                required => true, desc => ?DESC(auth_sasl_mechanism)
+            })},
+        {username, mk(binary(), #{required => true, desc => ?DESC(auth_sasl_username)})},
+        {password,
+            mk(binary(), #{required => true, sensitive => true, desc => ?DESC(auth_sasl_password)})}
+    ];
+fields(auth_gssapi_kerberos) ->
+    [
+        {kerberos_principal,
+            mk(binary(), #{
+                required => true,
+                desc => ?DESC(auth_kerberos_principal)
+            })},
+        {kerberos_keytab_file,
+            mk(binary(), #{
+                required => true,
+                desc => ?DESC(auth_kerberos_keytab_file)
+            })}
+    ];
+fields(socket_opts) ->
+    [
+        {sndbuf,
+            mk(
+                emqx_schema:bytesize(),
+                #{default => "1024KB", desc => ?DESC(socket_send_buffer)}
+            )},
+        {recbuf,
+            mk(
+                emqx_schema:bytesize(),
+                #{default => "1024KB", desc => ?DESC(socket_receive_buffer)}
+            )},
+        {nodelay,
+            mk(
+                boolean(),
+                #{default => true, desc => ?DESC(socket_nodelay)}
+            )}
+    ];
+fields(producer_opts) ->
+    [
+        {mqtt, mk(ref(producer_mqtt_opts), #{desc => ?DESC(producer_mqtt_opts)})},
+        {kafka,
+            mk(ref(producer_kafka_opts), #{
+                required => true,
+                desc => ?DESC(producer_kafka_opts)
+            })}
+    ];
+fields(producer_mqtt_opts) ->
+    [{topic, mk(string(), #{desc => ?DESC(mqtt_topic)})}];
+fields(producer_kafka_opts) ->
+    [
+        {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})},
+        {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})},
+        {max_batch_bytes,
+            mk(emqx_schema:bytesize(), #{default => "896KB", desc => ?DESC(max_batch_bytes)})},
+        {compression,
+            mk(enum([no_compression, snappy, gzip]), #{
+                default => no_compression, desc => ?DESC(compression)
+            })},
+        {partition_strategy,
+            mk(
+                enum([random, key_dispatch]),
+                #{default => random, desc => ?DESC(partition_strategy)}
+            )},
+        {required_acks,
+            mk(
+                enum([all_isr, leader_only, none]),
+                #{
+                    default => all_isr,
+                    desc => ?DESC(required_acks)
+                }
+            )},
+        {partition_count_refresh_interval,
+            mk(
+                emqx_schema:duration_s(),
+                #{
+                    default => "60s",
+                    desc => ?DESC(partition_count_refresh_interval)
+                }
+            )},
+        {max_inflight,
+            mk(
+                pos_integer(),
+                #{
+                    default => 10,
+                    desc => ?DESC(max_inflight)
+                }
+            )},
+        {buffer,
+            mk(ref(producer_buffer), #{
+                required => false,
+                desc => ?DESC(producer_buffer)
+            })}
+    ];
+fields(kafka_message) ->
+    [
+        {key, mk(string(), #{default => "${clientid}", desc => ?DESC(kafka_message_key)})},
+        {value, mk(string(), #{default => "${payload}", desc => ?DESC(kafka_message_value)})},
+        {timestamp,
+            mk(string(), #{
+                default => "${timestamp}", desc => ?DESC(kafka_message_timestamp)
+            })}
+    ];
+fields(producer_buffer) ->
+    [
+        {mode,
+            mk(
+                enum([memory, disk, hybrid]),
+                #{default => memory, desc => ?DESC(buffer_mode)}
+            )},
+        {per_partition_limit,
+            mk(
+                emqx_schema:bytesize(),
+                #{default => "2GB", desc => ?DESC(buffer_per_partition_limit)}
+            )},
+        {segment_bytes,
+            mk(
+                emqx_schema:bytesize(),
+                #{default => "100MB", desc => ?DESC(buffer_segment_bytes)}
+            )},
+        {memory_overload_protection,
+            mk(boolean(), #{
+                %% different from 4.x
+                default => true,
+                desc => ?DESC(buffer_memory_overload_protection)
+            })}
+    ].
+
+% fields(consumer_opts) ->
+%     [
+%         {kafka, mk(ref(consumer_kafka_opts), #{required => true, desc => ?DESC(consumer_kafka_opts)})},
+%         {mqtt, mk(ref(consumer_mqtt_opts), #{required => true, desc => ?DESC(consumer_mqtt_opts)})}
+%     ];
+% fields(consumer_mqtt_opts) ->
+%     [ {topic, mk(string(), #{desc => ?DESC(consumer_mqtt_topic)})}
+%     ];
+
+% fields(consumer_mqtt_opts) ->
+%     [ {topic, mk(string(), #{desc => ?DESC(consumer_mqtt_topic)})}
+%     ];
+% fields(consumer_kafka_opts) ->
+%     [ {topic, mk(string(), #{desc => ?DESC(consumer_kafka_topic)})}
+%     ].
+
+desc("config") ->
+    ?DESC("desc_config");
+desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
+    ["Configuration for Kafka using `", string:to_upper(Method), "` method."];
+desc(_) ->
+    undefined.
+
+%% -------------------------------------------------------------------------------------------------
+%% internal
+type_field() ->
+    {type, mk(enum([kafka]), #{required => true, desc => ?DESC("desc_type")})}.
+
+name_field() ->
+    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
+
+ref(Name) ->
+    hoconsc:ref(?MODULE, Name).
diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl
new file mode 100644
index 000000000..d1fad4765
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl
@@ -0,0 +1,33 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+%% Kafka connection configuration
+-module(emqx_bridge_impl_kafka).
+-behaviour(emqx_resource).
+
+%% callbacks of behaviour emqx_resource
+-export([
+    callback_mode/0,
+    on_start/2,
+    on_stop/2,
+    on_query/3,
+    on_get_status/2,
+    is_buffer_supported/0
+]).
+
+is_buffer_supported() -> true.
+
+callback_mode() -> async_if_possible.
+
+on_start(InstId, Config) ->
+    emqx_bridge_impl_kafka_producer:on_start(InstId, Config).
+
+on_stop(InstId, State) ->
+    emqx_bridge_impl_kafka_producer:on_stop(InstId, State).
+
+on_query(InstId, Msg, State) ->
+    emqx_bridge_impl_kafka_producer:on_query(InstId, Msg, State).
+
+on_get_status(InstId, State) ->
+    emqx_bridge_impl_kafka_producer:on_get_status(InstId, State).
diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
new file mode 100644
index 000000000..ce82dbe2d
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -0,0 +1,270 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_bridge_impl_kafka_producer).
+
+%% callbacks of behaviour emqx_resource
+-export([
+    callback_mode/0,
+    on_start/2,
+    on_stop/2,
+    on_query/3,
+    on_get_status/2
+]).
+
+-export([on_kafka_ack/3]).
+
+-include_lib("emqx/include/logger.hrl").
+
+callback_mode() -> async_if_possible.
+
+%% @doc Config schema is defined in emqx_ee_bridge_kafka.
+on_start(InstId, Config) ->
+    #{
+        bridge_name := BridgeName,
+        bootstrap_hosts := Hosts0,
+        connect_timeout := ConnTimeout,
+        metadata_request_timeout := MetaReqTimeout,
+        min_metadata_refresh_interval := MinMetaRefreshInterval,
+        socket_opts := SocketOpts,
+        authentication := Auth,
+        ssl := SSL
+    } = Config,
+    %% it's a bug if producer config is not found
+    %% the caller should not try to start a producer if
+    %% there is no producer config
+    ProducerConfigWrapper = get_required(producer, Config, no_kafka_producer_config),
+    ProducerConfig = get_required(kafka, ProducerConfigWrapper, no_kafka_producer_parameters),
+    MessageTemplate = get_required(message, ProducerConfig, no_kafka_message_template),
+    Hosts = hosts(Hosts0),
+    ClientId = make_client_id(BridgeName),
+    ClientConfig = #{
+        min_metadata_refresh_interval => MinMetaRefreshInterval,
+        connect_timeout => ConnTimeout,
+        client_id => ClientId,
+        request_timeout => MetaReqTimeout,
+        extra_sock_opts => socket_opts(SocketOpts),
+        sasl => sasl(Auth),
+        ssl => ssl(SSL)
+    },
+    #{
+        topic := KafkaTopic
+    } = ProducerConfig,
+    case wolff:ensure_supervised_client(ClientId, Hosts, ClientConfig) of
+        {ok, _} ->
+            ?SLOG(info, #{
+                msg => "kafka_client_started",
+                instance_id => InstId,
+                kafka_hosts => Hosts
+            });
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "failed_to_start_kafka_client",
+                instance_id => InstId,
+                kafka_hosts => Hosts,
+                reason => Reason
+            }),
+            throw(failed_to_start_kafka_client)
+    end,
+    WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig),
+    case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of
+        {ok, Producers} ->
+            {ok, #{
+                message_template => compile_message_template(MessageTemplate),
+                client_id => ClientId,
+                producers => Producers
+            }};
+        {error, Reason2} ->
+            ?SLOG(error, #{
+                msg => "failed_to_start_kafka_producer",
+                instance_id => InstId,
+                kafka_hosts => Hosts,
+                kafka_topic => KafkaTopic,
+                reason => Reason2
+            }),
+            throw(failed_to_start_kafka_producer)
+    end.
+
+on_stop(_InstId, #{client_id := ClientID, producers := Producers}) ->
+    with_log_at_error(
+        fun() -> wolff:stop_and_delete_supervised_producers(Producers) end,
+        #{
+            msg => "failed_to_delete_kafka_producer",
+            client_id => ClientID
+        }
+    ),
+    with_log_at_error(
+        fun() -> wolff:stop_and_delete_supervised_client(ClientID) end,
+        #{
+            msg => "failed_to_delete_kafka_client",
+            client_id => ClientID
+        }
+    ).
+
+%% @doc The callback API for rule-engine (or bridge without rules)
+%% The input argument `Message' is an enriched format (as a map())
+%% of the original #message{} record.
+%% The enrichment is done by rule-engine or by the data bridge framework.
+%% E.g. the output of rule-engine process chain
+%% or the direct mapping from an MQTT message.
+on_query(_InstId, {send_message, Message}, #{message_template := Template, producers := Producers}) ->
+    KafkaMessage = render_message(Template, Message),
+    %% The retuned information is discarded here.
+    %% If the producer process is down when sending, this function would
+    %% raise an error exception which is to be caught by the caller of this callback
+    {_Partition, _Pid} = wolff:send(Producers, [KafkaMessage], {fun ?MODULE:on_kafka_ack/3, [#{}]}),
+    ok.
+
+compile_message_template(#{
+    key := KeyTemplate, value := ValueTemplate, timestamp := TimestampTemplate
+}) ->
+    #{
+        key => emqx_plugin_libs_rule:preproc_tmpl(KeyTemplate),
+        value => emqx_plugin_libs_rule:preproc_tmpl(ValueTemplate),
+        timestamp => emqx_plugin_libs_rule:preproc_tmpl(TimestampTemplate)
+    }.
+
+render_message(
+    #{key := KeyTemplate, value := ValueTemplate, timestamp := TimestampTemplate}, Message
+) ->
+    #{
+        key => render(KeyTemplate, Message),
+        value => render(ValueTemplate, Message),
+        ts => render_timestamp(TimestampTemplate, Message)
+    }.
+
+render(Template, Message) ->
+    emqx_plugin_libs_rule:proc_tmpl(Template, Message).
+
+render_timestamp(Template, Message) ->
+    try
+        binary_to_integer(render(Template, Message))
+    catch
+        _:_ ->
+            erlang:system_time(millisecond)
+    end.
+
+on_kafka_ack(_Partition, _Offset, _Extra) ->
+    %% Do nothing so far.
+    %% Maybe need to bump some counters?
+    ok.
+
+on_get_status(_InstId, _State) ->
+    connected.
+
+%% Parse comma separated host:port list into a [{Host,Port}] list
+hosts(Hosts) when is_binary(Hosts) ->
+    hosts(binary_to_list(Hosts));
+hosts(Hosts) when is_list(Hosts) ->
+    kpro:parse_endpoints(Hosts).
+
+%% Extra socket options, such as sndbuf size etc.
+socket_opts(Opts) when is_map(Opts) ->
+    socket_opts(maps:to_list(Opts));
+socket_opts(Opts) when is_list(Opts) ->
+    socket_opts_loop(Opts, []).
+
+socket_opts_loop([], Acc) ->
+    lists:reverse(Acc);
+socket_opts_loop([{T, Bytes} | Rest], Acc) when
+    T =:= sndbuf orelse T =:= recbuf orelse T =:= buffer
+->
+    Acc1 = [{T, Bytes} | adjust_socket_buffer(Bytes, Acc)],
+    socket_opts_loop(Rest, Acc1);
+socket_opts_loop([Other | Rest], Acc) ->
+    socket_opts_loop(Rest, [Other | Acc]).
+
+%% https://www.erlang.org/doc/man/inet.html
+%% For TCP it is recommended to have val(buffer) >= val(recbuf)
+%% to avoid performance issues because of unnecessary copying.
+adjust_socket_buffer(Bytes, Opts) ->
+    case lists:keytake(buffer, 1, Opts) of
+        false ->
+            [{buffer, Bytes} | Opts];
+        {value, {buffer, Bytes1}, Acc1} ->
+            [{buffer, max(Bytes1, Bytes)} | Acc1]
+    end.
+
+sasl(none) ->
+    undefined;
+sasl(#{mechanism := Mechanism, username := Username, password := Password}) ->
+    {Mechanism, Username, emqx_secret:wrap(Password)};
+sasl(#{
+    kerberos_principal := Principal,
+    kerberos_keytab_file := KeyTabFile
+}) ->
+    {callback, brod_gssapi, {gssapi, KeyTabFile, Principal}}.
+
+ssl(#{enable := true} = SSL) ->
+    emqx_tls_lib:to_client_opts(SSL);
+ssl(_) ->
+    [].
+
+producers_config(BridgeName, ClientId, Input) ->
+    #{
+        max_batch_bytes := MaxBatchBytes,
+        compression := Compression,
+        partition_strategy := PartitionStrategy,
+        required_acks := RequiredAcks,
+        partition_count_refresh_interval := PCntRefreshInterval,
+        max_inflight := MaxInflight,
+        buffer := #{
+            mode := BufferMode,
+            per_partition_limit := PerPartitionLimit,
+            segment_bytes := SegmentBytes,
+            memory_overload_protection := MemOLP
+        }
+    } = Input,
+
+    {OffloadMode, ReplayqDir} =
+        case BufferMode of
+            memory -> {false, false};
+            disk -> {false, replayq_dir(ClientId)};
+            hybrid -> {true, replayq_dir(ClientId)}
+        end,
+    #{
+        name => make_producer_name(BridgeName),
+        partitioner => PartitionStrategy,
+        partition_count_refresh_interval_seconds => PCntRefreshInterval,
+        replayq_dir => ReplayqDir,
+        replayq_offload_mode => OffloadMode,
+        replayq_max_total_bytes => PerPartitionLimit,
+        replayq_seg_bytes => SegmentBytes,
+        drop_if_highmem => MemOLP,
+        required_acks => RequiredAcks,
+        max_batch_bytes => MaxBatchBytes,
+        max_send_ahead => MaxInflight - 1,
+        compression => Compression
+    }.
+
+replayq_dir(ClientId) ->
+    filename:join([emqx:data_dir(), "kafka", ClientId]).
+
+%% Client ID is better to be unique to make it easier for Kafka side trouble shooting.
+make_client_id(BridgeName) when is_atom(BridgeName) ->
+    make_client_id(atom_to_list(BridgeName));
+make_client_id(BridgeName) ->
+    iolist_to_binary([BridgeName, ":", atom_to_list(node())]).
+
+%% Producer name must be an atom which will be used as a ETS table name for
+%% partition worker lookup.
+make_producer_name(BridgeName) when is_atom(BridgeName) ->
+    make_producer_name(atom_to_list(BridgeName));
+make_producer_name(BridgeName) ->
+    list_to_atom("kafka_producer_" ++ BridgeName).
+
+with_log_at_error(Fun, Log) ->
+    try
+        Fun()
+    catch
+        C:E ->
+            ?SLOG(error, Log#{
+                exception => C,
+                reason => E
+            })
+    end.
+
+get_required(Field, Config, Throw) ->
+    Value = maps:get(Field, Config, none),
+    Value =:= none andalso throw(Throw),
+    Value.
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
new file mode 100644
index 000000000..8d01d0d69
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -0,0 +1,90 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_impl_kafka_producer_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("brod/include/brod.hrl").
+
+-define(PRODUCER, emqx_bridge_impl_kafka).
+
+%%------------------------------------------------------------------------------
+%% CT boilerplate
+%%------------------------------------------------------------------------------
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    {ok, _} = application:ensure_all_started(brod),
+    {ok, _} = application:ensure_all_started(wolff),
+    Config.
+
+end_per_suite(_) ->
+    ok.
+
+t_publish(_CtConfig) ->
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "kafka_hosts_string" => kafka_hosts_string(),
+        "kafka_topic" => KafkaTopic
+    }),
+    InstId = <<"InstanceID">>,
+    Time = erlang:system_time(millisecond),
+    BinTime = integer_to_binary(Time),
+    Msg = #{
+        clientid => BinTime,
+        payload => <<"payload">>,
+        timestamp => Time
+    },
+    {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0),
+    ct:pal("base offset before testing ~p", [Offset]),
+    {ok, State} = ?PRODUCER:on_start(InstId, Conf),
+    ok = ?PRODUCER:on_query(InstId, {send_message, Msg}, State),
+    {ok, {_, [KafkaMsg]}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset),
+    ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg),
+    ok = ?PRODUCER:on_stop(InstId, State),
+    ok.
+
+config(Args) ->
+    {ok, Conf} = hocon:binary(hocon_config(Args)),
+    #{config := Parsed} = hocon_tconf:check_plain(
+        emqx_ee_bridge_kafka,
+        #{<<"config">> => Conf},
+        #{atom_key => true}
+    ),
+    Parsed#{bridge_name => "testbridge"}.
+
+hocon_config(Args) ->
+    Hocon = bbmustache:render(iolist_to_binary(hocon_config_template()), Args),
+    Hocon.
+
+%% erlfmt-ignore
+hocon_config_template() ->
+"""
+bootstrap_hosts = \"{{ kafka_hosts_string }}\"
+enable = true
+authentication = none
+producer = {
+    mqtt {
+       topic = \"t/#\"
+    }
+    kafka = {
+        topic = \"{{ kafka_topic }}\"
+    }
+}
+""".
+
+kafka_hosts_string() ->
+    "kafka-1.emqx.net:9092,".
+
+kafka_hosts() ->
+    kpro:parse_endpoints(kafka_hosts_string()).
+
+resolve_kafka_offset(Hosts, Topic, Partition) ->
+    brod:resolve_offset(Hosts, Topic, Partition, latest).
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 675a934aa..c1b86d20b 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -5,7 +5,9 @@
         kernel,
         stdlib,
         hstreamdb_erl,
-        influxdb
+        influxdb,
+        wolff,
+        brod
     ]},
     {env, []},
     {modules, []},
diff --git a/mix.exs b/mix.exs
index d871ddf82..2b7f1419b 100644
--- a/mix.exs
+++ b/mix.exs
@@ -44,7 +44,7 @@ defmodule EMQXUmbrella.MixProject do
     # we need several overrides here because dependencies specify
     # other exact versions, and not ranges.
     [
-      {:lc, github: "emqx/lc", tag: "0.3.1"},
+      {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
       {:redbug, "2.0.7"},
       {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
       {:ehttpc, github: "emqx/ehttpc", tag: "0.4.0", override: true},
@@ -57,7 +57,7 @@ defmodule EMQXUmbrella.MixProject do
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.7", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true},
-      {:replayq, "0.3.4", override: true},
+      {:replayq, github: "emqx/replayq", tag: "0.3.4", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
       {:emqtt, github: "emqx/emqtt", tag: "1.7.0-rc.1", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
@@ -129,7 +129,13 @@ defmodule EMQXUmbrella.MixProject do
   defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
     [
       {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
-      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.3", override: true}
+      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.3", override: true},
+      {:wolff, github: "kafka4beam/wolff", tag: "1.6.4"},
+      {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.0", override: true},
+      {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"},
+      {:brod, github: "kafka4beam/brod", tag: "3.16.4"},
+      {:snappyer, "1.2.8", override: true},
+      {:supervisor3, "1.1.11", override: true}
     ]
   end
 
diff --git a/rebar.config b/rebar.config
index e29c5f2c7..e5bab7c6d 100644
--- a/rebar.config
+++ b/rebar.config
@@ -44,7 +44,7 @@
 {post_hooks,[]}.
 
 {deps,
-    [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}}
+    [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}
     , {redbug, "2.0.7"}
     , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}}
@@ -59,7 +59,7 @@
     , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.7"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
-    , {replayq, "0.3.4"}
+    , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.4"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0-rc.1"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 2c87bb0cf..c478ce005 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -10,12 +10,20 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
 help() {
     echo
     echo "-h|--help:              To display this usage info"
-    echo "--app lib_dir/app_name: Print apps in json"
+    echo "--app lib_dir/app_name: For which app to run start docker-compose, and run common tests"
+    echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl"
     echo "--console:              Start EMQX in console mode"
+    echo "--attach:               Attach to the Erlang docker container without running any test case"
+    echo "--only-up:              Keep the testbed running after CT"
+    echo "--keep-up:              Only start the testbed but do not run CT"
 }
 
 WHICH_APP='novalue'
 CONSOLE='no'
+KEEP_UP='no'
+ONLY_UP='no'
+SUITES=''
+ATTACH='no'
 while [ "$#" -gt 0 ]; do
     case $1 in
         -h|--help)
@@ -26,10 +34,26 @@ while [ "$#" -gt 0 ]; do
             WHICH_APP="$2"
             shift 2
             ;;
+        --only-up)
+            ONLY_UP='yes'
+            shift 1
+            ;;
+        --keep-up)
+            KEEP_UP='yes'
+            shift 1
+            ;;
+        --attach)
+            ATTACH='yes'
+            shift 1
+            ;;
         --console)
             CONSOLE='yes'
             shift 1
             ;;
+        --suites)
+            SUITES="$2"
+            shift 2
+            ;;
         *)
             echo "unknown option $1"
             exit 1
@@ -45,6 +69,16 @@ fi
 ERLANG_CONTAINER='erlang24'
 DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct"
 
+case "${WHICH_APP}" in
+    lib-ee*)
+        ## ensure enterprise profile when testing lib-ee applications
+        export PROFILE='emqx-enterprise'
+        ;;
+    *)
+        true
+        ;;
+esac
+
 if [ -f "$DOCKER_CT_ENVS_FILE" ]; then
     # shellcheck disable=SC2002
     CT_DEPS="$(cat "$DOCKER_CT_ENVS_FILE" | xargs)"
@@ -80,6 +114,9 @@ for dep in ${CT_DEPS}; do
             FILES+=( '.ci/docker-compose-file/docker-compose-pgsql-tcp.yaml'
                      '.ci/docker-compose-file/docker-compose-pgsql-tls.yaml' )
             ;;
+        kafka)
+            FILES+=( '.ci/docker-compose-file/docker-compose-kafka.yaml' )
+            ;;
         *)
             echo "unknown_ct_dependency $dep"
             exit 1
@@ -104,13 +141,23 @@ if [[ -t 1 ]]; then
 fi
 docker exec -i $TTY "$ERLANG_CONTAINER" bash -c 'git config --global --add safe.directory /emqx'
 
-if [ "$CONSOLE" = 'yes' ]; then
+if [ "$ONLY_UP" = 'yes' ]; then
+    exit 0
+fi
+
+if [ "$ATTACH" = 'yes' ]; then
+    docker exec -it "$ERLANG_CONTAINER" bash
+elif [ "$CONSOLE" = 'yes' ]; then
     docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
 else
     set +e
-    docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct"
+    docker exec -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct"
     RESULT=$?
-    # shellcheck disable=2086 # no quotes for F_OPTIONS
-    docker-compose $F_OPTIONS down
-    exit $RESULT
+    if [ "$KEEP_UP" = 'yes' ]; then
+        exit $RESULT
+    else
+        # shellcheck disable=2086 # no quotes for F_OPTIONS
+        docker-compose $F_OPTIONS down
+        exit $RESULT
+    fi
 fi
diff --git a/scripts/find-suites.sh b/scripts/find-suites.sh
index 4d2fd3bee..e7c1b422e 100755
--- a/scripts/find-suites.sh
+++ b/scripts/find-suites.sh
@@ -8,5 +8,9 @@ set -euo pipefail
 # ensure dir
 cd -P -- "$(dirname -- "$0")/.."
 
-TESTDIR="$1/test"
-find "${TESTDIR}" -name "*_SUITE.erl" -print0 2>/dev/null | xargs -0 | tr ' ' ','
+if [ -z "${EMQX_CT_SUITES:-}" ]; then
+    TESTDIR="$1/test"
+    find "${TESTDIR}" -name "*_SUITE.erl" -print0 2>/dev/null | xargs -0 | tr ' ' ','
+else
+    echo "${EMQX_CT_SUITES}"
+fi

From f0e03086a639390df54b5698d68ce2a1cbcb9a59 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Mon, 12 Sep 2022 20:54:58 +0200
Subject: [PATCH 140/232] test: add test cases for Kafka SASL auth mechanisms
 plain and scram

---
 .../docker-compose-kafka.yaml                 | 15 +++-
 .ci/docker-compose-file/kafka/jaas.conf       |  9 +++
 .../kafka/run_add_scram_users.sh              | 26 ++++++
 .../emqx_bridge_impl_kafka_producer_SUITE.erl | 80 +++++++++++++++++--
 scripts/ct/run.sh                             |  6 +-
 5 files changed, 122 insertions(+), 14 deletions(-)
 create mode 100644 .ci/docker-compose-file/kafka/jaas.conf
 create mode 100755 .ci/docker-compose-file/kafka/run_add_scram_users.sh

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index 85532725d..edde553af 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -13,15 +13,24 @@ services:
     image: wurstmeister/kafka:2.13-2.7.0
     ports:
       - "9092:9092"
+      - "9093:9093"
     container_name: kafka-1.emqx.net
     hostname: kafka-1.emqx.net
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENERS: PLAINTEXT://:9092
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+      KAFKA_LISTENERS: PLAINTEXT://:9092,SASL_PLAINTEXT://:9093
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
+      KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
+      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+
       KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
     networks:
       emqx_bridge:
+    volumes:
+      - ./kafka/jaas.conf:/etc/kafka/jaas.conf
+      - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh
+    command: run_add_scram_users.sh
diff --git a/.ci/docker-compose-file/kafka/jaas.conf b/.ci/docker-compose-file/kafka/jaas.conf
new file mode 100644
index 000000000..bf6e6716b
--- /dev/null
+++ b/.ci/docker-compose-file/kafka/jaas.conf
@@ -0,0 +1,9 @@
+KafkaServer {
+   org.apache.kafka.common.security.plain.PlainLoginModule required
+   user_admin="password"
+   user_emqxuser="password";
+
+   org.apache.kafka.common.security.scram.ScramLoginModule required
+   username="admin"
+   password="password";
+};
diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
new file mode 100755
index 000000000..3a3d2ee21
--- /dev/null
+++ b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "+++++++ Starting Kafka ++++++++"
+
+start-kafka.sh &
+
+SERVER=localhost
+PORT1=9092
+PORT2=9093
+TIMEOUT=60
+
+echo "+++++++ Wait until Kafka ports are up ++++++++"
+
+timeout $TIMEOUT bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1
+
+timeout $TIMEOUT bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT2
+
+echo "+++++++ Run config commands ++++++++"
+
+kafka-configs.sh --bootstrap-server localhost:9092 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=password],SCRAM-SHA-512=[password=password]' --entity-type users --entity-name emqxuser
+
+echo "+++++++ Wait until Kafka ports are down ++++++++"
+
+bash -c 'while printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 8d01d0d69..0eb393d4d 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -28,12 +28,7 @@ init_per_suite(Config) ->
 end_per_suite(_) ->
     ok.
 
-t_publish(_CtConfig) ->
-    KafkaTopic = "test-topic-one-partition",
-    Conf = config(#{
-        "kafka_hosts_string" => kafka_hosts_string(),
-        "kafka_topic" => KafkaTopic
-    }),
+do_publish(Conf, KafkaTopic) ->
     InstId = <<"InstanceID">>,
     Time = erlang:system_time(millisecond),
     BinTime = integer_to_binary(Time),
@@ -51,6 +46,54 @@ t_publish(_CtConfig) ->
     ok = ?PRODUCER:on_stop(InstId, State),
     ok.
 
+t_publish(_CtConfig) ->
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "authentication" => "none",
+        "kafka_hosts_string" => kafka_hosts_string(),
+        "kafka_topic" => KafkaTopic
+    }),
+    do_publish(Conf, KafkaTopic).
+
+t_publish_sasl_plain(_CtConfig) ->
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "authentication" => #{
+            "mechanism" => "plain",
+            "username" => "emqxuser",
+            "password" => "password"
+        },
+        "kafka_hosts_string" => kafka_hosts_string_sasl(),
+        "kafka_topic" => KafkaTopic
+    }),
+    do_publish(Conf, KafkaTopic).
+
+t_publish_sasl_scram256(_CtConfig) ->
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "authentication" => #{
+            "mechanism" => "scram_sha_256",
+            "username" => "emqxuser",
+            "password" => "password"
+        },
+        "kafka_hosts_string" => kafka_hosts_string_sasl(),
+        "kafka_topic" => KafkaTopic
+    }),
+    do_publish(Conf, KafkaTopic).
+
+t_publish_sasl_scram512(_CtConfig) ->
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "authentication" => #{
+            "mechanism" => "scram_sha_512",
+            "username" => "emqxuser",
+            "password" => "password"
+        },
+        "kafka_hosts_string" => kafka_hosts_string_sasl(),
+        "kafka_topic" => KafkaTopic
+    }),
+    do_publish(Conf, KafkaTopic).
+
 config(Args) ->
     {ok, Conf} = hocon:binary(hocon_config(Args)),
     #{config := Parsed} = hocon_tconf:check_plain(
@@ -61,7 +104,13 @@ config(Args) ->
     Parsed#{bridge_name => "testbridge"}.
 
 hocon_config(Args) ->
-    Hocon = bbmustache:render(iolist_to_binary(hocon_config_template()), Args),
+    AuthConf = maps:get("authentication", Args),
+    AuthTemplate = iolist_to_binary(hocon_config_template_authentication(AuthConf)),
+    AuthConfRendered = bbmustache:render(AuthTemplate, AuthConf),
+    Hocon = bbmustache:render(
+        iolist_to_binary(hocon_config_template()),
+        Args#{"authentication" => AuthConfRendered}
+    ),
     Hocon.
 
 %% erlfmt-ignore
@@ -69,7 +118,7 @@ hocon_config_template() ->
 """
 bootstrap_hosts = \"{{ kafka_hosts_string }}\"
 enable = true
-authentication = none
+authentication = {{{ authentication }}} 
 producer = {
     mqtt {
        topic = \"t/#\"
@@ -80,9 +129,24 @@ producer = {
 }
 """.
 
+%% erlfmt-ignore
+hocon_config_template_authentication("none") ->
+    "none";
+hocon_config_template_authentication(#{"mechanism" := _}) ->
+"""
+{
+    mechanism = {{ mechanism }}
+    password = {{ password }}
+    username = {{ username }}
+}
+""".
+
 kafka_hosts_string() ->
     "kafka-1.emqx.net:9092,".
 
+kafka_hosts_string_sasl() ->
+    "kafka-1.emqx.net:9093,".
+
 kafka_hosts() ->
     kpro:parse_endpoints(kafka_hosts_string()).
 
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index c478ce005..45d32767c 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -14,8 +14,8 @@ help() {
     echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl"
     echo "--console:              Start EMQX in console mode"
     echo "--attach:               Attach to the Erlang docker container without running any test case"
-    echo "--only-up:              Keep the testbed running after CT"
-    echo "--keep-up:              Only start the testbed but do not run CT"
+    echo "--only-up:              Only start the testbed but do not run CT"
+    echo "--keep-up:              Keep the testbed running after CT"
 }
 
 WHICH_APP='novalue'
@@ -151,7 +151,7 @@ elif [ "$CONSOLE" = 'yes' ]; then
     docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
 else
     set +e
-    docker exec -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct"
+    docker exec -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
     RESULT=$?
     if [ "$KEEP_UP" = 'yes' ]; then
         exit $RESULT

From e45c99bf79dd1ee1bce6a0947531ffa1183565a0 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 12 Sep 2022 10:51:35 +0200
Subject: [PATCH 141/232] fix: kafka bridge schema

---
 .../i18n/emqx_ee_bridge_kafka.conf              |  8 ++++----
 .../emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl | 17 +++++++++++++++--
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
index a47bf1249..1fdbfedc4 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
@@ -219,7 +219,7 @@ emqx_ee_bridge_kafka {
     socket_nodelay {
         desc {
             en: "When set to 'true', TCP buffer sent as soon as possible. "
-                "Otherwise the OS kernel may buffer small TCP packets for a while (40ms by default)."
+                "Otherwise, the OS kernel may buffer small TCP packets for a while (40 ms by default)."
             zh: "设置 ‘true' 让系统内核立即发送。否则当需要发送当内容很少时，可能会有一定延迟（默认 40 毫秒）。"
         }
         label {
@@ -294,12 +294,12 @@ emqx_ee_bridge_kafka {
     max_batch_bytes {
         desc {
             en: "Maximum bytes to collect in a Kafka message batch. "
-                "Most of the Kafka brokers default to a limit of 1MB batch size. "
-                "EMQX's default value is less than 1MB in order to compensate "
+                "Most of the Kafka brokers default to a limit of 1 MB batch size. "
+                "EMQX's default value is less than 1 MB in order to compensate "
                 "Kafka message encoding overheads (especially when each individual message is very small). "
                 "When a single message is over the limit, it is still sent (as a single element batch)."
             zh: "最大消息批量字节数。"
-                "大多数 Kafka 环境的默认最低值是 1MB，EMQX 的默认值比 1MB 更小是因为需要"
+                "大多数 Kafka 环境的默认最低值是 1 MB，EMQX 的默认值比 1 MB 更小是因为需要"
                 "补偿 Kafka 消息编码索需要的额外字节（尤其是当每条消息都很小的情况下）。"
                 "当单个消息的大小超过该限制时，它仍然会被发送，（相当于该批量中只有单个消息）。"
         }
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
index 080af35d0..ac5177f6e 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
@@ -245,8 +245,21 @@ desc("config") ->
     ?DESC("desc_config");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for Kafka using `", string:to_upper(Method), "` method."];
-desc(_) ->
-    undefined.
+desc(Name) ->
+    lists:member(Name, struct_names()) orelse throw({missing_desc, Name}),
+    ?DESC(Name).
+
+struct_names() ->
+    [
+        auth_gssapi_kerberos,
+        auth_username_password,
+        kafka_message,
+        producer_buffer,
+        producer_kafka_opts,
+        producer_mqtt_opts,
+        socket_opts,
+        producer_opts
+    ].
 
 %% -------------------------------------------------------------------------------------------------
 %% internal

From 499ed3ca51af97a7780ff529888a3142c26f6026 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Wed, 14 Sep 2022 09:57:34 +0800
Subject: [PATCH 142/232] chore: don't generate enterprise.conf when not
 emqx-enterprise profile

---
 scripts/merge-config.escript | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript
index f617dbb70..f78083ee1 100755
--- a/scripts/merge-config.escript
+++ b/scripts/merge-config.escript
@@ -12,13 +12,13 @@
 -define(APPS, ["emqx", "emqx_dashboard", "emqx_authz"]).
 
 main(_) ->
-    Profile = os:getenv("PROFILE", "emqx"),
     {ok, BaseConf} = file:read_file("apps/emqx_conf/etc/emqx_conf.conf"),
     Cfgs = get_all_cfgs("apps/"),
+    IsEnterprise = is_enterprise(),
     Enterprise =
-        case Profile of
-            "emqx" -> [];
-            "emqx-enterprise" -> [io_lib:nl(), "include emqx-enterprise.conf", io_lib:nl()]
+        case IsEnterprise of
+            false -> [];
+            true -> [io_lib:nl(), "include emqx-enterprise.conf", io_lib:nl()]
         end,
     Conf = [
         merge(BaseConf, Cfgs),
@@ -27,10 +27,18 @@ main(_) ->
     ],
     ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", Conf),
 
-    EnterpriseCfgs = get_all_cfgs("lib-ee/"),
-    EnterpriseConf = merge("", EnterpriseCfgs),
+    case IsEnterprise of
+        true ->
+            EnterpriseCfgs = get_all_cfgs("lib-ee/"),
+            EnterpriseConf = merge("", EnterpriseCfgs),
+            ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf);
+        false ->
+            ok
+    end.
 
-    ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf).
+is_enterprise() ->
+    Profile = os:getenv("PROFILE", "emqx"),
+    nomatch =/= string:find(Profile, "enterprise").
 
 merge(BaseConf, Cfgs) ->
     lists:foldl(

From f41adb099794d85c917f5ad0496e164c0e5fb9ac Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 14 Sep 2022 15:18:07 +0800
Subject: [PATCH 143/232] refactor: change some default values of resource_opts

---
 apps/emqx_bridge/src/emqx_bridge_api.erl               | 4 ++--
 apps/emqx_resource/include/emqx_resource.hrl           | 8 ++++----
 apps/emqx_resource/src/schema/emqx_resource_schema.erl | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl  | 4 ++--
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl     | 2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index a353c9cf0..6d5ecf808 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -208,8 +208,8 @@ info_example_basic(webhook) ->
             auto_restart_interval => 15000,
             query_mode => async,
             async_inflight_window => 100,
-            enable_queue => true,
-            max_queue_bytes => 1024 * 1024 * 1024
+            enable_queue => false,
+            max_queue_bytes => 100 * 1024 * 1024
         }
     };
 info_example_basic(mqtt) ->
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index aab0129d1..71300df72 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -84,15 +84,15 @@
 -define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024).
 -define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>).
 
--define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024 * 1024).
--define(DEFAULT_QUEUE_SIZE_RAW, <<"100GB">>).
+-define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024).
+-define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>).
 
 %% count
 -define(DEFAULT_BATCH_SIZE, 100).
 
 %% milliseconds
--define(DEFAULT_BATCH_TIME, 10).
--define(DEFAULT_BATCH_TIME_RAW, <<"10ms">>).
+-define(DEFAULT_BATCH_TIME, 20).
+-define(DEFAULT_BATCH_TIME_RAW, <<"20ms">>).
 
 %% count
 -define(DEFAULT_INFLIGHT, 100).
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index df284bbe8..c666974b1 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -82,7 +82,7 @@ query_mode(_) -> undefined.
 
 enable_batch(type) -> boolean();
 enable_batch(required) -> false;
-enable_batch(default) -> false;
+enable_batch(default) -> true;
 enable_batch(desc) -> ?DESC("enable_batch");
 enable_batch(_) -> undefined.
 
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 6ad804b2c..a2f125722 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -93,8 +93,8 @@ values(common, Protocol, SupportUint, TypeOpts) ->
         precision => ms,
         resource_opts => #{
             enable_batch => false,
-            batch_size => 5,
-            batch_time => <<"1m">>
+            batch_size => 100,
+            batch_time => <<"20ms">>
         },
         server => <<"127.0.0.1:8086">>,
         ssl => #{enable => false}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
index 616842292..bdbf96424 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
@@ -58,7 +58,7 @@ values(post) ->
             worker_pool_size => 1,
             health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
             auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
-            enable_batch => false,
+            enable_batch => true,
             batch_size => ?DEFAULT_BATCH_SIZE,
             batch_time => ?DEFAULT_BATCH_TIME,
             query_mode => async,

From 1c03c236f5cfcac14befb6c17b34a93ec5408f42 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 14 Sep 2022 15:19:30 +0800
Subject: [PATCH 144/232] fix(mqtt_bridge): handle send_to_remote in idle state

---
 apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl | 6 +++++-
 apps/emqx_resource/src/emqx_resource_manager.erl            | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index 618361ad3..5f81e68e1 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -276,6 +276,8 @@ idle({call, From}, ensure_started, State) ->
         {error, Reason, _State} ->
             {keep_state_and_data, [{reply, From, {error, Reason}}]}
     end;
+idle({call, From}, send_to_remote, _State) ->
+    {keep_state_and_data, [{reply, From, {error, {recoverable_error, not_connected}}}]};
 %% @doc Standing by for manual start.
 idle(info, idle, #{start_type := manual}) ->
     keep_state_and_data;
@@ -339,10 +341,12 @@ common(_StateName, {call, From}, get_forwards, #{connect_opts := #{forwards := F
     {keep_state_and_data, [{reply, From, Forwards}]};
 common(_StateName, {call, From}, get_subscriptions, #{connection := Connection}) ->
     {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, #{})}]};
+common(_StateName, {call, From}, Req, _State) ->
+    {keep_state_and_data, [{reply, From, {unsuppored_request, Req}}]};
 common(_StateName, info, {'EXIT', _, _}, State) ->
     {keep_state, State};
 common(StateName, Type, Content, #{name := Name} = State) ->
-    ?SLOG(notice, #{
+    ?SLOG(error, #{
         msg => "bridge_discarded_event",
         name => Name,
         type => Type,
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index e4ba92b5c..a896e990e 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -467,11 +467,11 @@ retry_actions(Data) ->
 
 handle_remove_event(From, ClearMetrics, Data) ->
     stop_resource(Data),
+    ok = emqx_resource_worker_sup:stop_workers(Data#data.id, Data#data.opts),
     case ClearMetrics of
         true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id);
         false -> ok
     end,
-    ok = emqx_resource_worker_sup:stop_workers(Data#data.id, Data#data.opts),
     {stop_and_reply, normal, [{reply, From, ok}]}.
 
 start_resource(Data, From) ->

From 5820b028cb263002a7937ad637200b537c909397 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Wed, 14 Sep 2022 15:52:27 +0200
Subject: [PATCH 145/232] feat: add test case for Kerberos Kafka authentication

---
 .../docker-compose-kafka.yaml                 | 24 ++++++++++++++++--
 .ci/docker-compose-file/docker-compose.yaml   |  6 +++++
 .ci/docker-compose-file/kafka/jaas.conf       |  7 ++++++
 .../kafka/run_add_scram_users.sh              |  9 +++++++
 .ci/docker-compose-file/kerberos/krb5.conf    | 23 +++++++++++++++++
 .ci/docker-compose-file/kerberos/run.sh       | 25 +++++++++++++++++++
 .../emqx_bridge_impl_kafka_producer_SUITE.erl | 19 ++++++++++++++
 7 files changed, 111 insertions(+), 2 deletions(-)
 create mode 100644 .ci/docker-compose-file/kerberos/krb5.conf
 create mode 100755 .ci/docker-compose-file/kerberos/run.sh

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index edde553af..a4a064c16 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -16,6 +16,8 @@ services:
       - "9093:9093"
     container_name: kafka-1.emqx.net
     hostname: kafka-1.emqx.net
+    depends_on:
+      - "kdc" 
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
@@ -23,14 +25,32 @@ services:
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
+      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
+      KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
-
       KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
+      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
     networks:
       emqx_bridge:
     volumes:
+      - emqx-shared-secret:/var/lib/secret
       - ./kafka/jaas.conf:/etc/kafka/jaas.conf
       - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh
+      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
+      - ./kerberos/krb5.conf:/etc/krb5.conf
     command: run_add_scram_users.sh
+  kdc:
+    hostname: kdc.emqx.net
+    image:  ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
+    container_name: kdc.emqx.net
+    networks:
+      emqx_bridge:
+    volumes:
+      - emqx-shared-secret:/var/lib/secret
+      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
+      - ./kerberos/krb5.conf:/etc/krb5.conf
+      - ./kerberos/run.sh:/usr/bin/run.sh
+    command: run.sh
+
diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index 2612eb8d8..8db53d562 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -18,6 +18,9 @@ services:
       - emqx_bridge
     volumes:
       - ../..:/emqx
+      - emqx-shared-secret:/var/lib/secret
+      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
+      - ./kerberos/krb5.conf:/etc/krb5.conf
     working_dir: /emqx
     tty: true
 
@@ -33,3 +36,6 @@ networks:
           gateway: 172.100.239.1
         - subnet: 2001:3200:3200::/64
           gateway: 2001:3200:3200::1
+
+volumes:   # add this section
+  emqx-shared-secret:    # does not need anything underneath this
diff --git a/.ci/docker-compose-file/kafka/jaas.conf b/.ci/docker-compose-file/kafka/jaas.conf
index bf6e6716b..f6158950e 100644
--- a/.ci/docker-compose-file/kafka/jaas.conf
+++ b/.ci/docker-compose-file/kafka/jaas.conf
@@ -6,4 +6,11 @@ KafkaServer {
    org.apache.kafka.common.security.scram.ScramLoginModule required
    username="admin"
    password="password";
+
+   com.sun.security.auth.module.Krb5LoginModule required
+   useKeyTab=true
+   storeKey=true
+   keyTab="/var/lib/secret/kafka.key"
+   principal="kafka/kafka-1.emqx.net@KDC.EMQX.NET";
+
 };
diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
index 3a3d2ee21..1ffb900a8 100755
--- a/.ci/docker-compose-file/kafka/run_add_scram_users.sh
+++ b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
@@ -2,6 +2,15 @@
 
 set -euo pipefail
 
+
+TIMEOUT=60
+
+echo "+++++++ Wait until Kerberos Keytab is created ++++++++"
+
+timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.key ]; do sleep 1; done'
+
+sleep 3
+
 echo "+++++++ Starting Kafka ++++++++"
 
 start-kafka.sh &
diff --git a/.ci/docker-compose-file/kerberos/krb5.conf b/.ci/docker-compose-file/kerberos/krb5.conf
new file mode 100644
index 000000000..032236888
--- /dev/null
+++ b/.ci/docker-compose-file/kerberos/krb5.conf
@@ -0,0 +1,23 @@
+[libdefaults]
+  default_realm = KDC.EMQX.NET
+  ticket_lifetime = 24h
+  renew_lifetime = 7d
+  forwardable = true
+  rdns = false
+  dns_lookup_kdc   = no
+  dns_lookup_realm = no
+
+[realms]
+  KDC.EMQX.NET = {
+    kdc = kdc
+    admin_server = kadmin
+  }
+
+[domain_realm]
+  kdc.emqx.net = KDC.EMQX.NET
+  .kdc.emqx.net = KDC.EMQX.NET
+
+[logging]
+  kdc = FILE:/var/log/kerberos/krb5kdc.log
+  admin_server = FILE:/var/log/kerberos/kadmin.log
+  default = FILE:/var/log/kerberos/krb5lib.log
diff --git a/.ci/docker-compose-file/kerberos/run.sh b/.ci/docker-compose-file/kerberos/run.sh
new file mode 100755
index 000000000..c5547fb59
--- /dev/null
+++ b/.ci/docker-compose-file/kerberos/run.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+
+echo "Remove old keytabs"
+
+rm -f /var/lib/secret/kafka.key 2>&1 > /dev/null
+rm -f /var/lib/secret/rig.key 2>&1 > /dev/null
+
+echo "Create realm"
+
+kdb5_util -P emqx -r KDC.EMQX.NET create -s
+
+echo "Add principals"
+
+kadmin.local -w password -q "add_principal -randkey kafka/kafka-1.emqx.net@KDC.EMQX.NET"
+kadmin.local -w password -q "add_principal -randkey rig@KDC.EMQX.NET"  > /dev/null
+
+
+echo "Create keytabs"
+
+kadmin.local -w password -q "ktadd  -k /var/lib/secret/kafka.key -norandkey kafka/kafka-1.emqx.net@KDC.EMQX.NET " > /dev/null
+kadmin.local -w password -q "ktadd  -k /var/lib/secret/rig.key -norandkey rig@KDC.EMQX.NET " > /dev/null
+
+echo STARTING KDC
+/usr/sbin/krb5kdc -n
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 0eb393d4d..a767a18e4 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -94,6 +94,18 @@ t_publish_sasl_scram512(_CtConfig) ->
     }),
     do_publish(Conf, KafkaTopic).
 
+t_publish_sasl_kerberos(_CtConfig) ->
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "authentication" => #{
+            "kerberos_principal" => "rig@KDC.EMQX.NET",
+            "kerberos_keytab_file" => "/var/lib/secret/rig.key"
+        },
+        "kafka_hosts_string" => kafka_hosts_string_sasl(),
+        "kafka_topic" => KafkaTopic
+    }),
+    do_publish(Conf, KafkaTopic).
+
 config(Args) ->
     {ok, Conf} = hocon:binary(hocon_config(Args)),
     #{config := Parsed} = hocon_tconf:check_plain(
@@ -139,6 +151,13 @@ hocon_config_template_authentication(#{"mechanism" := _}) ->
     password = {{ password }}
     username = {{ username }}
 }
+""";
+hocon_config_template_authentication(#{"kerberos_principal" := _}) ->
+"""
+{
+    kerberos_principal = \"{{ kerberos_principal }}\"
+    kerberos_keytab_file = \"{{ kerberos_keytab_file }}\"
+}
 """.
 
 kafka_hosts_string() ->

From 4e211c12d3db5e074313c6b6dcfc481f4964471d Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 14 Sep 2022 16:15:10 +0800
Subject: [PATCH 146/232] fix(mqtt_bridge): return value of sending messages
 was discarded

---
 apps/emqx_connector/src/emqx_connector_mqtt.erl            | 6 ++----
 .../emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl | 2 +-
 apps/emqx_resource/src/emqx_resource_worker.erl            | 7 ++++++-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index f52edab15..b063d7436 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -187,8 +187,7 @@ on_stop(_InstId, #{name := InstanceId}) ->
 
 on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) ->
     ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
-    emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
-    ok.
+    emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg).
 
 on_query_async(
     _InstId,
@@ -197,8 +196,7 @@ on_query_async(
     #{name := InstanceId}
 ) ->
     ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
-    emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, {ReplayFun, Args}),
-    ok.
+    emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, {ReplayFun, Args}).
 
 on_get_status(_InstId, #{name := InstanceId, bridge_conf := Conf}) ->
     AutoReconn = maps:get(auto_reconnect, Conf, true),
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index 5f81e68e1..e1651f114 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -342,7 +342,7 @@ common(_StateName, {call, From}, get_forwards, #{connect_opts := #{forwards := F
 common(_StateName, {call, From}, get_subscriptions, #{connection := Connection}) ->
     {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, #{})}]};
 common(_StateName, {call, From}, Req, _State) ->
-    {keep_state_and_data, [{reply, From, {unsuppored_request, Req}}]};
+    {keep_state_and_data, [{reply, From, {error, {unsuppored_request, Req}}}]};
 common(_StateName, info, {'EXIT', _, _}, State) ->
     {keep_state, State};
 common(StateName, Type, Content, #{name := Name} = State) ->
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index f683b67ed..05292944b 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -585,7 +585,12 @@ assert_ok_result(ok) ->
 assert_ok_result({async_return, R}) ->
     assert_ok_result(R);
 assert_ok_result(R) when is_tuple(R) ->
-    ok = erlang:element(1, R);
+    try
+        ok = erlang:element(1, R)
+    catch
+        error:{badmatch, _} ->
+            error({not_ok_result, R})
+    end;
 assert_ok_result(R) ->
     error({not_ok_result, R}).
 

From 54a9c8d2018cf27800e07771973521c4fecd3b5a Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 5 Sep 2022 16:43:15 +0800
Subject: [PATCH 147/232] chore: refine influxdb bridge description

---
 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
index f34de1119..b2c3c5a73 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf
@@ -65,7 +65,7 @@ TLDR: </br>
     desc_type {
         desc {
             en: """The Bridge Type."""
-            zh: """Bridge 类型。"""
+            zh: """桥接类型。"""
         }
         label {
             en: "Bridge Type"
@@ -75,12 +75,12 @@ TLDR: </br>
 
     desc_name {
         desc {
-            en: """Bridge name, used as a human-readable description of the bridge."""
-            zh: """桥接名字，人类可读的描述信息。"""
+            en: """Bridge name."""
+            zh: """桥接名称。"""
         }
         label {
             en: "Bridge Name"
-            zh: "桥接名字"
+            zh: "桥接名称"
         }
     }
 

From 0390a5e547324bf0f2ffffb5c6f2e81d151d77e1 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 5 Sep 2022 18:18:53 +0800
Subject: [PATCH 148/232] fix(bridge): mysql bridge error case

---
 apps/emqx_bridge/src/emqx_bridge_api.erl            | 11 +++++++++++
 apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl | 13 +++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index a353c9cf0..d5edb1203 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -532,6 +532,17 @@ lookup_from_local_node(BridgeType, BridgeName) ->
                                 {200};
                             {error, timeout} ->
                                 {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+                            {error, {start_pool_failed, Name, Reason}} ->
+                                {503,
+                                    error_msg(
+                                        'SERVICE_UNAVAILABLE',
+                                        bin(
+                                            io_lib:format(
+                                                "failed to start ~p pool for reason ~p",
+                                                [Name, Reason]
+                                            )
+                                        )
+                                    )};
                             {error, Reason} ->
                                 {500, error_msg('INTERNAL_ERROR', Reason)}
                         end
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl
index cce45fa4a..f0b1f0bb5 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl
@@ -40,12 +40,13 @@ start_pool(Name, Mod, Options) ->
             stop_pool(Name),
             start_pool(Name, Mod, Options);
         {error, Reason} ->
+            NReason = parse_reason(Reason),
             ?SLOG(error, #{
                 msg => "start_ecpool_error",
                 pool_name => Name,
-                reason => Reason
+                reason => NReason
             }),
-            {error, {start_pool_failed, Name, Reason}}
+            {error, {start_pool_failed, Name, NReason}}
     end.
 
 stop_pool(Name) ->
@@ -86,3 +87,11 @@ health_check_ecpool_workers(PoolName, CheckFunc, Timeout) when is_function(Check
         exit:timeout ->
             false
     end.
+
+parse_reason({
+    {shutdown, {failed_to_start_child, _, {shutdown, {failed_to_start_child, _, Reason}}}},
+    _
+}) ->
+    Reason;
+parse_reason(Reason) ->
+    Reason.

From d5d3972ff5c318d4f4604876e9e30b3ec8c854c2 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 15 Sep 2022 08:57:17 +0800
Subject: [PATCH 149/232] chore: add test cases for MQTT Bridge reconnecting

---
 .../test/emqx_bridge_mqtt_SUITE.erl           | 116 ++++++++++++++++++
 .../src/mqtt/emqx_connector_mqtt_worker.erl   |   4 +-
 .../src/emqx_resource_manager.erl             |   2 +-
 .../src/emqx_resource_worker.erl              |   3 +-
 4 files changed, 121 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
index e35ad5fe5..c89621d59 100644
--- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -481,5 +481,121 @@ t_egress_mqtt_bridge_with_rules(_) ->
     {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []).
 
+t_mqtt_conn_bridge_egress_reconnect(_) ->
+    %% then we add a mqtt connector, using POST
+    User1 = <<"user1">>,
+
+    {ok, 201, Bridge} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF(User1)#{
+            <<"type">> => ?TYPE_MQTT,
+            <<"name">> => ?BRIDGE_NAME_EGRESS,
+            <<"egress">> => ?EGRESS_CONF,
+            %% to make it reconnect quickly
+            <<"reconnect_interval">> => <<"1s">>,
+            <<"resource_opts">> => #{
+                <<"worker_pool_size">> => 2,
+                <<"enable_queue">> => true,
+                <<"query_mode">> => <<"sync">>,
+                %% to make it check the healthy quickly
+                <<"health_check_interval">> => <<"0.5s">>
+            }
+        }
+    ),
+    #{
+        <<"type">> := ?TYPE_MQTT,
+        <<"name">> := ?BRIDGE_NAME_EGRESS
+    } = jsx:decode(Bridge),
+    BridgeIDEgress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_EGRESS),
+    %% we now test if the bridge works as expected
+    LocalTopic = <<"local_topic/1">>,
+    RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
+    Payload = <<"hello">>,
+    emqx:subscribe(RemoteTopic),
+    timer:sleep(100),
+    %% PUBLISH a message to the 'local' broker, as we have only one broker,
+    %% the remote broker is also the local one.
+    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+
+    %% we should receive a message on the "remote" broker, with specified topic
+    assert_mqtt_msg_received(RemoteTopic, Payload),
+
+    %% verify the metrics of the bridge
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(
+        #{
+            <<"metrics">> := #{<<"matched">> := 1, <<"success">> := 1, <<"failed">> := 0},
+            <<"node_metrics">> :=
+                [
+                    #{
+                        <<"node">> := _,
+                        <<"metrics">> :=
+                            #{<<"matched">> := 1, <<"success">> := 1, <<"failed">> := 0}
+                    }
+                ]
+        },
+        jsx:decode(BridgeStr)
+    ),
+
+    %% stop the listener 1883 to make the bridge disconnected
+    ok = emqx_listeners:stop_listener('tcp:default'),
+
+    %% PUBLISH 2 messages to the 'local' broker, the message should
+    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+
+    %% verify the metrics of the bridge, the message should be queued
+    {ok, 200, BridgeStr1} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(
+        #{
+            <<"status">> := Status,
+            <<"metrics">> := #{
+                <<"matched">> := 3, <<"success">> := 1, <<"failed">> := 0, <<"queuing">> := 2
+            }
+        } when Status == <<"connected">> orelse Status == <<"connecting">>,
+        jsx:decode(BridgeStr1)
+    ),
+
+    %% start the listener 1883 to make the bridge reconnected
+    ok = emqx_listeners:start_listener('tcp:default'),
+    timer:sleep(1500),
+    %% verify the metrics of the bridge, the 2 queued messages should have been sent
+    {ok, 200, BridgeStr2} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(
+        #{
+            <<"status">> := <<"connected">>,
+            <<"metrics">> := #{
+                <<"matched">> := 3,
+                <<"success">> := 3,
+                <<"failed">> := 0,
+                <<"queuing">> := 0,
+                <<"retried">> := R
+            }
+        } when R > 0,
+        jsx:decode(BridgeStr2)
+    ),
+    %% also verify the 2 messages have been sent to the remote broker
+    assert_mqtt_msg_received(RemoteTopic, Payload),
+    assert_mqtt_msg_received(RemoteTopic, Payload),
+    %% delete the bridge
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+    ok.
+
+assert_mqtt_msg_received(Topic, Payload) ->
+    ?assert(
+        receive
+            {deliver, Topic, #message{payload = Payload}} ->
+                ct:pal("Got mqtt message: ~p on topic ~p", [Payload, Topic]),
+                true;
+            Msg ->
+                ct:pal("Unexpected Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end
+    ).
+
 request(Method, Url, Body) ->
     request(<<"connector_admin">>, Method, Url, Body).
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index e1651f114..fe359437c 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -276,7 +276,7 @@ idle({call, From}, ensure_started, State) ->
         {error, Reason, _State} ->
             {keep_state_and_data, [{reply, From, {error, Reason}}]}
     end;
-idle({call, From}, send_to_remote, _State) ->
+idle({call, From}, {send_to_remote, _}, _State) ->
     {keep_state_and_data, [{reply, From, {error, {recoverable_error, not_connected}}}]};
 %% @doc Standing by for manual start.
 idle(info, idle, #{start_type := manual}) ->
@@ -342,7 +342,7 @@ common(_StateName, {call, From}, get_forwards, #{connect_opts := #{forwards := F
 common(_StateName, {call, From}, get_subscriptions, #{connection := Connection}) ->
     {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, #{})}]};
 common(_StateName, {call, From}, Req, _State) ->
-    {keep_state_and_data, [{reply, From, {error, {unsuppored_request, Req}}}]};
+    {keep_state_and_data, [{reply, From, {error, {unsupported_request, Req}}}]};
 common(_StateName, info, {'EXIT', _, _}, State) ->
     {keep_state, State};
 common(StateName, Type, Content, #{name := Name} = State) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index a896e990e..3430b3964 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -525,7 +525,7 @@ handle_connecting_health_check(Data) ->
             (connected, UpdatedData) ->
                 {next_state, connected, UpdatedData};
             (connecting, UpdatedData) ->
-                Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}],
+                Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}],
                 {keep_state, UpdatedData, Actions};
             (disconnected, UpdatedData) ->
                 {next_state, disconnected, UpdatedData}
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 05292944b..1f6c4f599 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -133,6 +133,7 @@ init({Id, Index, Opts}) ->
         end,
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', queue_count(Queue)),
     ok = inflight_new(Name),
+    HCItvl = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL),
     St = #{
         id => Id,
         index => Index,
@@ -142,7 +143,7 @@ init({Id, Index, Opts}) ->
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
         queue => Queue,
-        resume_interval => maps:get(resume_interval, Opts, ?HEALTHCHECK_INTERVAL),
+        resume_interval => maps:get(resume_interval, Opts, HCItvl),
         acc => [],
         acc_left => BatchSize,
         tref => undefined

From 4dc26eeba79711aa792733dc2efa1411cf3a8a90 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 15 Sep 2022 07:33:07 +0200
Subject: [PATCH 150/232] fix: use different instance id in Kafka auth test

---
 .../test/emqx_bridge_impl_kafka_producer_SUITE.erl  | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index a767a18e4..775942015 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -28,8 +28,7 @@ init_per_suite(Config) ->
 end_per_suite(_) ->
     ok.
 
-do_publish(Conf, KafkaTopic) ->
-    InstId = <<"InstanceID">>,
+do_publish(Conf, KafkaTopic, InstId) ->
     Time = erlang:system_time(millisecond),
     BinTime = integer_to_binary(Time),
     Msg = #{
@@ -53,7 +52,7 @@ t_publish(_CtConfig) ->
         "kafka_hosts_string" => kafka_hosts_string(),
         "kafka_topic" => KafkaTopic
     }),
-    do_publish(Conf, KafkaTopic).
+    do_publish(Conf, KafkaTopic, <<"NoAuthInst">>).
 
 t_publish_sasl_plain(_CtConfig) ->
     KafkaTopic = "test-topic-one-partition",
@@ -66,7 +65,7 @@ t_publish_sasl_plain(_CtConfig) ->
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
         "kafka_topic" => KafkaTopic
     }),
-    do_publish(Conf, KafkaTopic).
+    do_publish(Conf, KafkaTopic, <<"SASLPlainInst">>).
 
 t_publish_sasl_scram256(_CtConfig) ->
     KafkaTopic = "test-topic-one-partition",
@@ -79,7 +78,7 @@ t_publish_sasl_scram256(_CtConfig) ->
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
         "kafka_topic" => KafkaTopic
     }),
-    do_publish(Conf, KafkaTopic).
+    do_publish(Conf, KafkaTopic, <<"SASLScram256Inst">>).
 
 t_publish_sasl_scram512(_CtConfig) ->
     KafkaTopic = "test-topic-one-partition",
@@ -92,7 +91,7 @@ t_publish_sasl_scram512(_CtConfig) ->
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
         "kafka_topic" => KafkaTopic
     }),
-    do_publish(Conf, KafkaTopic).
+    do_publish(Conf, KafkaTopic, <<"SASLScram512Inst">>).
 
 t_publish_sasl_kerberos(_CtConfig) ->
     KafkaTopic = "test-topic-one-partition",
@@ -104,7 +103,7 @@ t_publish_sasl_kerberos(_CtConfig) ->
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
         "kafka_topic" => KafkaTopic
     }),
-    do_publish(Conf, KafkaTopic).
+    do_publish(Conf, KafkaTopic, <<"SASLKerberosInst">>).
 
 config(Args) ->
     {ok, Conf} = hocon:binary(hocon_config(Args)),

From be7a8c11a8227a572e361f3b1cac9a3852a15f6c Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 15 Sep 2022 16:21:32 +0200
Subject: [PATCH 151/232] test: make bridge name unique in tests

---
 .../emqx_bridge_impl_kafka_producer_SUITE.erl | 40 +++++++++++++------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 775942015..1c79e9bd8 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -38,23 +38,28 @@ do_publish(Conf, KafkaTopic, InstId) ->
     },
     {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0),
     ct:pal("base offset before testing ~p", [Offset]),
-    {ok, State} = ?PRODUCER:on_start(InstId, Conf),
-    ok = ?PRODUCER:on_query(InstId, {send_message, Msg}, State),
+    StartRes = ?PRODUCER:on_start(InstId, Conf),
+    {ok, State} = StartRes,
+    OnQueryRes = ?PRODUCER:on_query(InstId, {send_message, Msg}, State),
+    ok = OnQueryRes,
     {ok, {_, [KafkaMsg]}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset),
     ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg),
     ok = ?PRODUCER:on_stop(InstId, State),
     ok.
 
 t_publish(_CtConfig) ->
+    InstId = emqx_bridge_resource:resource_id("kafka", "NoAuthInst"),
     KafkaTopic = "test-topic-one-partition",
     Conf = config(#{
         "authentication" => "none",
         "kafka_hosts_string" => kafka_hosts_string(),
-        "kafka_topic" => KafkaTopic
+        "kafka_topic" => KafkaTopic,
+        "instance_id" => InstId
     }),
-    do_publish(Conf, KafkaTopic, <<"NoAuthInst">>).
+    do_publish(Conf, KafkaTopic, InstId).
 
 t_publish_sasl_plain(_CtConfig) ->
+    InstId = emqx_bridge_resource:resource_id("kafka", "SASLPlainInst"),
     KafkaTopic = "test-topic-one-partition",
     Conf = config(#{
         "authentication" => #{
@@ -63,11 +68,14 @@ t_publish_sasl_plain(_CtConfig) ->
             "password" => "password"
         },
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic
+        "kafka_topic" => KafkaTopic,
+        "instance_id" => InstId
     }),
-    do_publish(Conf, KafkaTopic, <<"SASLPlainInst">>).
+    do_publish(Conf, KafkaTopic, InstId).
 
 t_publish_sasl_scram256(_CtConfig) ->
+    InstId = emqx_bridge_resource:resource_id("kafka", "SASLScram256Inst"),
+    KafkaTopic = "test-topic-one-partition",
     KafkaTopic = "test-topic-one-partition",
     Conf = config(#{
         "authentication" => #{
@@ -76,11 +84,13 @@ t_publish_sasl_scram256(_CtConfig) ->
             "password" => "password"
         },
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic
+        "kafka_topic" => KafkaTopic,
+        "instance_id" => InstId
     }),
-    do_publish(Conf, KafkaTopic, <<"SASLScram256Inst">>).
+    do_publish(Conf, KafkaTopic, InstId).
 
 t_publish_sasl_scram512(_CtConfig) ->
+    InstId = emqx_bridge_resource:resource_id("kafka", "SASLScram512Inst"),
     KafkaTopic = "test-topic-one-partition",
     Conf = config(#{
         "authentication" => #{
@@ -89,11 +99,13 @@ t_publish_sasl_scram512(_CtConfig) ->
             "password" => "password"
         },
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic
+        "kafka_topic" => KafkaTopic,
+        "instance_id" => InstId
     }),
-    do_publish(Conf, KafkaTopic, <<"SASLScram512Inst">>).
+    do_publish(Conf, KafkaTopic, InstId).
 
 t_publish_sasl_kerberos(_CtConfig) ->
+    InstId = emqx_bridge_resource:resource_id("kafka", "SASLKerberosInst"),
     KafkaTopic = "test-topic-one-partition",
     Conf = config(#{
         "authentication" => #{
@@ -101,9 +113,10 @@ t_publish_sasl_kerberos(_CtConfig) ->
             "kerberos_keytab_file" => "/var/lib/secret/rig.key"
         },
         "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic
+        "kafka_topic" => KafkaTopic,
+        "instance_id" => InstId
     }),
-    do_publish(Conf, KafkaTopic, <<"SASLKerberosInst">>).
+    do_publish(Conf, KafkaTopic, InstId).
 
 config(Args) ->
     {ok, Conf} = hocon:binary(hocon_config(Args)),
@@ -112,7 +125,8 @@ config(Args) ->
         #{<<"config">> => Conf},
         #{atom_key => true}
     ),
-    Parsed#{bridge_name => "testbridge"}.
+    InstId = maps:get("instance_id", Args),
+    Parsed#{bridge_name => erlang:element(2, emqx_bridge_resource:parse_bridge_id(InstId))}.
 
 hocon_config(Args) ->
     AuthConf = maps:get("authentication", Args),

From 8307f04c2efe145f48fc6df9ac7734a4a5bc73d5 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 16 Sep 2022 16:52:08 +0800
Subject: [PATCH 152/232] refactor(resource): save inflight size into the ETS
 table

---
 .../src/emqx_resource_worker.erl              | 39 +++++++++----------
 .../test/emqx_resource_SUITE.erl              | 18 ++-------
 2 files changed, 23 insertions(+), 34 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 1f6c4f599..25032c29c 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -132,13 +132,13 @@ init({Id, Index, Opts}) ->
                 undefined
         end,
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', queue_count(Queue)),
-    ok = inflight_new(Name),
+    InfltWinSZ = maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT),
+    ok = inflight_new(Name, InfltWinSZ),
     HCItvl = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL),
     St = #{
         id => Id,
         index => Index,
         name => Name,
-        async_inflight_window => maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT),
         enable_batch => maps:get(enable_batch, Opts, false),
         batch_size => BatchSize,
         batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
@@ -288,8 +288,7 @@ query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left
     end;
 query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id} = St) ->
     QueryOpts = #{
-        inflight_name => maps:get(name, St),
-        inflight_window => maps:get(async_inflight_window, St)
+        inflight_name => maps:get(name, St)
     },
     Result = call_query(configured, Id, ?QUERY(From, Request), QueryOpts),
     case reply_caller(Id, ?REPLY(From, Request, Result)) of
@@ -312,8 +311,7 @@ flush(
 ) ->
     Batch = lists:reverse(Batch0),
     QueryOpts = #{
-        inflight_name => maps:get(name, St),
-        inflight_window => maps:get(async_inflight_window, St)
+        inflight_name => maps:get(name, St)
     },
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'batching', -length(Batch)),
     Result = call_query(configured, Id, Batch, QueryOpts),
@@ -464,12 +462,10 @@ apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) -
 apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
-    WinSize = maps:get(inflight_window, QueryOpts, undefined),
     ?APPLY_RESOURCE(
         call_query_async,
-        case inflight_is_full(Name, WinSize) of
+        case inflight_is_full(Name) of
             true ->
-                ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
                 ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight'),
@@ -489,12 +485,10 @@ apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
 apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
-    WinSize = maps:get(inflight_window, QueryOpts, undefined),
     ?APPLY_RESOURCE(
         call_batch_query_async,
-        case inflight_is_full(Name, WinSize) of
+        case inflight_is_full(Name) of
             true ->
-                ?tp(warning, inflight_full, #{id => Id, wind_size => WinSize}),
                 {async_return, inflight_full};
             false ->
                 BatchLen = length(Batch),
@@ -540,27 +534,32 @@ batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
     end.
 %%==============================================================================
 %% the inflight queue for async query
-
-inflight_new(Name) ->
+-define(SIZE_REF, -1).
+inflight_new(Name, InfltWinSZ) ->
     _ = ets:new(Name, [named_table, ordered_set, public, {write_concurrency, true}]),
+    inflight_append(Name, ?SIZE_REF, {size, InfltWinSZ}),
     ok.
 
 inflight_get_first(Name) ->
-    case ets:first(Name) of
+    case ets:next(Name, ?SIZE_REF) of
         '$end_of_table' ->
             empty;
         Ref ->
             case ets:lookup(Name, Ref) of
-                [Object] -> Object;
-                [] -> inflight_get_first(Name)
+                [Object] ->
+                    Object;
+                [] ->
+                    %% it might have been dropped
+                    inflight_get_first(Name)
             end
     end.
 
-inflight_is_full(undefined, _) ->
+inflight_is_full(undefined) ->
     false;
-inflight_is_full(Name, MaxSize) ->
+inflight_is_full(Name) ->
+    [{_, {size, MaxSize}}] = ets:lookup(Name, ?SIZE_REF),
     case ets:info(Name, size) of
-        Size when Size >= MaxSize -> true;
+        Size when Size > MaxSize -> true;
         _ -> false
     end.
 
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 3f42850ad..a9d274168 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -242,7 +242,7 @@ t_query_counter_async_query(_) ->
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
         #{name => test_resource, register => true},
-        #{query_mode => async}
+        #{query_mode => async, enable_batch => false}
     ),
     ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)),
     ?check_trace(
@@ -284,7 +284,7 @@ t_query_counter_async_callback(_) ->
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
         #{name => test_resource, register => true},
-        #{query_mode => async, async_inflight_window => 1000000}
+        #{query_mode => async, enable_batch => false, async_inflight_window => 1000000}
     ),
     ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)),
     ?check_trace(
@@ -338,6 +338,7 @@ t_query_counter_async_inflight(_) ->
         #{name => test_resource, register => true},
         #{
             query_mode => async,
+            enable_batch => false,
             async_inflight_window => WindowSize,
             worker_pool_size => 1,
             resume_interval => 300
@@ -358,7 +359,7 @@ t_query_counter_async_inflight(_) ->
         end
     ),
 
-    %% this will block the resource_worker
+    %% this will block the resource_worker as the inflight windown is full now
     ok = emqx_resource:query(?ID, {inc_counter, 1}),
     ?assertMatch(0, ets:info(Tab0, size)),
     %% sleep to make the resource_worker resume some times
@@ -683,17 +684,6 @@ inc_counter_in_parallel(N, Opts) ->
      || Pid <- Pids
     ].
 
-% verify_inflight_full(WindowSize) ->
-%     ?check_trace(
-%         ?TRACE_OPTS,
-%         emqx_resource:query(?ID, {inc_counter, 1}),
-%         fun(Return, Trace) ->
-%             QueryTrace = ?of_kind(inflight_full, Trace),
-%             ?assertMatch([#{wind_size := WindowSize} | _], QueryTrace),
-%             ?assertMatch(ok, Return)
-%         end
-%     ).
-
 bin_config() ->
     <<"\"name\": \"test_resource\"">>.
 

From ac2922fc4cd3c1f4980508df83d2cb6ab2d1cd70 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 16 Sep 2022 16:44:12 +0200
Subject: [PATCH 153/232] test: Kafka bridge cases for all combinations of SASL
 and SSL

---
 .../docker-compose-kafka.yaml                 |  72 +++---
 .../kafka/generate-certs.sh                   |  45 ++++
 .ci/docker-compose-file/kafka/jaas.conf       |   2 +-
 .../kafka/run_add_scram_users.sh              |  11 +-
 .ci/docker-compose-file/kerberos/run.sh       |   8 +-
 .../emqx_bridge_impl_kafka_producer_SUITE.erl | 209 +++++++++++-------
 6 files changed, 236 insertions(+), 111 deletions(-)
 create mode 100755 .ci/docker-compose-file/kafka/generate-certs.sh

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index a4a064c16..3bb7748d5 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -9,38 +9,13 @@ services:
     hostname: zookeeper
     networks:
       emqx_bridge:
-  kafka_1:
-    image: wurstmeister/kafka:2.13-2.7.0
-    ports:
-      - "9092:9092"
-      - "9093:9093"
-    container_name: kafka-1.emqx.net
-    hostname: kafka-1.emqx.net
-    depends_on:
-      - "kdc" 
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENERS: PLAINTEXT://:9092,SASL_PLAINTEXT://:9093
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
-      KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
-      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
-      KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
-      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
-      KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
-      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
-    networks:
-      emqx_bridge:
+  ssl_cert_gen:
+    image: fredrikhgrelland/alpine-jdk11-openssl
+    container_name: ssl_cert_gen
     volumes:
       - emqx-shared-secret:/var/lib/secret
-      - ./kafka/jaas.conf:/etc/kafka/jaas.conf
-      - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh
-      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
-      - ./kerberos/krb5.conf:/etc/krb5.conf
-    command: run_add_scram_users.sh
+      - ./kafka/generate-certs.sh:/bin/generate-certs.sh
+    command: /bin/generate-certs.sh
   kdc:
     hostname: kdc.emqx.net
     image:  ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
@@ -53,4 +28,41 @@ services:
       - ./kerberos/krb5.conf:/etc/krb5.conf
       - ./kerberos/run.sh:/usr/bin/run.sh
     command: run.sh
+  kafka_1:
+    image: wurstmeister/kafka:2.13-2.7.0
+    ports:
+      - "9092:9092"
+      - "9093:9093"
+    container_name: kafka-1.emqx.net
+    hostname: kafka-1.emqx.net
+    depends_on:
+      - "kdc"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENERS: PLAINTEXT://:9092,SASL_PLAINTEXT://:9093,SSL://:9094,SASL_SSL://:9095
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093,SSL://kafka-1.emqx.net:9094,SASL_SSL://kafka-1.emqx.net:9095
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
+      KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
+      KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
+      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+      KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
+      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
+      KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks
+      KAFKA_SSL_TRUSTSTORE_PASSWORD: password
+      KAFKA_SSL_KEYSTORE_LOCATION: /var/lib/secret/kafka.keystore.jks
+      KAFKA_SSL_KEYSTORE_PASSWORD: password
+      KAFKA_SSL_KEY_PASSWORD: password
+    networks:
+      emqx_bridge:
+    volumes:
+      - emqx-shared-secret:/var/lib/secret
+      - ./kafka/jaas.conf:/etc/kafka/jaas.conf
+      - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh
+      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
+      - ./kerberos/krb5.conf:/etc/krb5.conf
+    command: run_add_scram_users.sh
 
diff --git a/.ci/docker-compose-file/kafka/generate-certs.sh b/.ci/docker-compose-file/kafka/generate-certs.sh
new file mode 100755
index 000000000..d0ae4a8d0
--- /dev/null
+++ b/.ci/docker-compose-file/kafka/generate-certs.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+set -euo pipefail
+
+set -x
+
+# Source https://github.com/zmstone/docker-kafka/blob/master/generate-certs.sh
+
+HOST="*."
+DAYS=3650
+PASS="password"
+
+cd /var/lib/secret/
+
+# Delete old files
+(rm ca.key ca.crt server.key server.csr server.crt client.key client.csr client.crt server.p12 kafka.keystore.jks kafka.truststore.jks 2>/dev/null || true)
+
+ls
+
+echo == Generate self-signed server and client certificates
+echo = generate CA
+openssl req -new -x509 -keyout ca.key -out ca.crt -days $DAYS -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
+
+echo = generate server certificate request
+openssl req -newkey rsa:2048 -sha256 -keyout server.key -out server.csr -days $DAYS -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
+
+echo = sign server certificate
+openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days $DAYS -CAcreateserial
+
+echo = generate client certificate request
+openssl req -newkey rsa:2048 -sha256 -keyout client.key -out client.csr -days $DAYS -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
+
+echo == sign client certificate
+openssl x509 -req -CA ca.crt -CAkey ca.key -in client.csr -out client.crt -days $DAYS -CAserial ca.srl
+
+echo = Convert self-signed certificate to PKCS#12 format
+openssl pkcs12 -export -name $HOST -in server.crt -inkey server.key -out server.p12 -CAfile ca.crt -passout pass:$PASS
+
+echo = Import PKCS#12 into a java keystore
+
+echo $PASS | keytool -importkeystore -destkeystore kafka.keystore.jks -srckeystore server.p12 -srcstoretype pkcs12 -alias $HOST -storepass $PASS
+
+echo = Import CA into java truststore
+
+echo yes | keytool -keystore kafka.truststore.jks -alias CARoot -import -file ca.crt -storepass $PASS
diff --git a/.ci/docker-compose-file/kafka/jaas.conf b/.ci/docker-compose-file/kafka/jaas.conf
index f6158950e..8ffe8457d 100644
--- a/.ci/docker-compose-file/kafka/jaas.conf
+++ b/.ci/docker-compose-file/kafka/jaas.conf
@@ -10,7 +10,7 @@ KafkaServer {
    com.sun.security.auth.module.Krb5LoginModule required
    useKeyTab=true
    storeKey=true
-   keyTab="/var/lib/secret/kafka.key"
+   keyTab="/var/lib/secret/kafka.keytab"
    principal="kafka/kafka-1.emqx.net@KDC.EMQX.NET";
 
 };
diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
index 1ffb900a8..32f42a9e9 100755
--- a/.ci/docker-compose-file/kafka/run_add_scram_users.sh
+++ b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
@@ -5,9 +5,18 @@ set -euo pipefail
 
 TIMEOUT=60
 
+echo "+++++++ Sleep for a while to make sure that old keytab and truststore is deleted ++++++++"
+
+sleep 5
+
 echo "+++++++ Wait until Kerberos Keytab is created ++++++++"
 
-timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.key ]; do sleep 1; done'
+timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.keytab ]; do sleep 1; done'
+
+
+echo "+++++++ Wait until SSL certs are generated ++++++++"
+
+timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.truststore.jks ]; do sleep 1; done'
 
 sleep 3
 
diff --git a/.ci/docker-compose-file/kerberos/run.sh b/.ci/docker-compose-file/kerberos/run.sh
index c5547fb59..85f172207 100755
--- a/.ci/docker-compose-file/kerberos/run.sh
+++ b/.ci/docker-compose-file/kerberos/run.sh
@@ -3,8 +3,8 @@
 
 echo "Remove old keytabs"
 
-rm -f /var/lib/secret/kafka.key 2>&1 > /dev/null
-rm -f /var/lib/secret/rig.key 2>&1 > /dev/null
+rm -f /var/lib/secret/kafka.keytab 2>&1 > /dev/null
+rm -f /var/lib/secret/rig.keytab 2>&1 > /dev/null
 
 echo "Create realm"
 
@@ -18,8 +18,8 @@ kadmin.local -w password -q "add_principal -randkey rig@KDC.EMQX.NET"  > /dev/nu
 
 echo "Create keytabs"
 
-kadmin.local -w password -q "ktadd  -k /var/lib/secret/kafka.key -norandkey kafka/kafka-1.emqx.net@KDC.EMQX.NET " > /dev/null
-kadmin.local -w password -q "ktadd  -k /var/lib/secret/rig.key -norandkey rig@KDC.EMQX.NET " > /dev/null
+kadmin.local -w password -q "ktadd  -k /var/lib/secret/kafka.keytab -norandkey kafka/kafka-1.emqx.net@KDC.EMQX.NET " > /dev/null
+kadmin.local -w password -q "ktadd  -k /var/lib/secret/rig.keytab -norandkey rig@KDC.EMQX.NET " > /dev/null
 
 echo STARTING KDC
 /usr/sbin/krb5kdc -n
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 1c79e9bd8..9ca87d106 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -20,16 +20,84 @@
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
+wait_until_kafka_is_up() ->
+    wait_until_kafka_is_up(0).
+
+wait_until_kafka_is_up(90) ->
+    ct:fail("Kafka is not up even though we have waited for a while");
+wait_until_kafka_is_up(Attempts) ->
+    KafkaTopic = "test-topic-one-partition",
+    case resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0) of
+        {ok, _} ->
+            ok;
+        _ ->
+            timer:sleep(1000),
+            wait_until_kafka_is_up(Attempts + 1)
+    end.
+
 init_per_suite(Config) ->
     {ok, _} = application:ensure_all_started(brod),
     {ok, _} = application:ensure_all_started(wolff),
+    wait_until_kafka_is_up(),
     Config.
 
 end_per_suite(_) ->
     ok.
 
-do_publish(Conf, KafkaTopic, InstId) ->
-    Time = erlang:system_time(millisecond),
+t_publish_no_auth(_CtConfig) ->
+    publish_with_and_without_ssl("none").
+
+t_publish_sasl_plain(_CtConfig) ->
+    publish_with_and_without_ssl(valid_sasl_plain_settings()).
+
+t_publish_sasl_scram256(_CtConfig) ->
+    publish_with_and_without_ssl(valid_sasl_scram256_settings()).
+
+t_publish_sasl_scram512(_CtConfig) ->
+    publish_with_and_without_ssl(valid_sasl_scram512_settings()).
+
+t_publish_sasl_kerberos(_CtConfig) ->
+    publish_with_and_without_ssl(valid_sasl_kerberos_settings()).
+
+publish_with_and_without_ssl(AuthSettings) ->
+    publish_helper(#{
+        auth_settings => AuthSettings,
+        ssl_settings => #{}
+    }),
+    publish_helper(#{
+        auth_settings => AuthSettings,
+        ssl_settings => valid_ssl_settings()
+    }).
+
+publish_helper(#{
+    auth_settings := AuthSettings,
+    ssl_settings := SSLSettings
+}) ->
+    HostsString =
+        case {AuthSettings, SSLSettings} of
+            {"none", Map} when map_size(Map) =:= 0 ->
+                kafka_hosts_string();
+            {"none", Map} when map_size(Map) =/= 0 ->
+                kafka_hosts_string_ssl();
+            {_, Map} when map_size(Map) =:= 0 ->
+                kafka_hosts_string_sasl();
+            {_, _} ->
+                kafka_hosts_string_ssl_sasl()
+        end,
+    Hash = erlang:phash2([HostsString, AuthSettings, SSLSettings]),
+    Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash),
+    InstId = emqx_bridge_resource:resource_id("kafka", Name),
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "authentication" => AuthSettings,
+        "kafka_hosts_string" => HostsString,
+        "kafka_topic" => KafkaTopic,
+        "instance_id" => InstId,
+        "ssl" => SSLSettings
+    }),
+    %% To make sure we get unique value
+    timer:sleep(1),
+    Time = erlang:monotonic_time(),
     BinTime = integer_to_binary(Time),
     Msg = #{
         clientid => BinTime,
@@ -47,79 +115,10 @@ do_publish(Conf, KafkaTopic, InstId) ->
     ok = ?PRODUCER:on_stop(InstId, State),
     ok.
 
-t_publish(_CtConfig) ->
-    InstId = emqx_bridge_resource:resource_id("kafka", "NoAuthInst"),
-    KafkaTopic = "test-topic-one-partition",
-    Conf = config(#{
-        "authentication" => "none",
-        "kafka_hosts_string" => kafka_hosts_string(),
-        "kafka_topic" => KafkaTopic,
-        "instance_id" => InstId
-    }),
-    do_publish(Conf, KafkaTopic, InstId).
-
-t_publish_sasl_plain(_CtConfig) ->
-    InstId = emqx_bridge_resource:resource_id("kafka", "SASLPlainInst"),
-    KafkaTopic = "test-topic-one-partition",
-    Conf = config(#{
-        "authentication" => #{
-            "mechanism" => "plain",
-            "username" => "emqxuser",
-            "password" => "password"
-        },
-        "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic,
-        "instance_id" => InstId
-    }),
-    do_publish(Conf, KafkaTopic, InstId).
-
-t_publish_sasl_scram256(_CtConfig) ->
-    InstId = emqx_bridge_resource:resource_id("kafka", "SASLScram256Inst"),
-    KafkaTopic = "test-topic-one-partition",
-    KafkaTopic = "test-topic-one-partition",
-    Conf = config(#{
-        "authentication" => #{
-            "mechanism" => "scram_sha_256",
-            "username" => "emqxuser",
-            "password" => "password"
-        },
-        "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic,
-        "instance_id" => InstId
-    }),
-    do_publish(Conf, KafkaTopic, InstId).
-
-t_publish_sasl_scram512(_CtConfig) ->
-    InstId = emqx_bridge_resource:resource_id("kafka", "SASLScram512Inst"),
-    KafkaTopic = "test-topic-one-partition",
-    Conf = config(#{
-        "authentication" => #{
-            "mechanism" => "scram_sha_512",
-            "username" => "emqxuser",
-            "password" => "password"
-        },
-        "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic,
-        "instance_id" => InstId
-    }),
-    do_publish(Conf, KafkaTopic, InstId).
-
-t_publish_sasl_kerberos(_CtConfig) ->
-    InstId = emqx_bridge_resource:resource_id("kafka", "SASLKerberosInst"),
-    KafkaTopic = "test-topic-one-partition",
-    Conf = config(#{
-        "authentication" => #{
-            "kerberos_principal" => "rig@KDC.EMQX.NET",
-            "kerberos_keytab_file" => "/var/lib/secret/rig.key"
-        },
-        "kafka_hosts_string" => kafka_hosts_string_sasl(),
-        "kafka_topic" => KafkaTopic,
-        "instance_id" => InstId
-    }),
-    do_publish(Conf, KafkaTopic, InstId).
-
 config(Args) ->
-    {ok, Conf} = hocon:binary(hocon_config(Args)),
+    ConfText = hocon_config(Args),
+    ct:pal("Running tests with conf:\n~s", [ConfText]),
+    {ok, Conf} = hocon:binary(ConfText),
     #{config := Parsed} = hocon_tconf:check_plain(
         emqx_ee_bridge_kafka,
         #{<<"config">> => Conf},
@@ -132,9 +131,15 @@ hocon_config(Args) ->
     AuthConf = maps:get("authentication", Args),
     AuthTemplate = iolist_to_binary(hocon_config_template_authentication(AuthConf)),
     AuthConfRendered = bbmustache:render(AuthTemplate, AuthConf),
+    SSLConf = maps:get("ssl", Args, #{}),
+    SSLTemplate = iolist_to_binary(hocon_config_template_ssl(SSLConf)),
+    SSLConfRendered = bbmustache:render(SSLTemplate, SSLConf),
     Hocon = bbmustache:render(
         iolist_to_binary(hocon_config_template()),
-        Args#{"authentication" => AuthConfRendered}
+        Args#{
+            "authentication" => AuthConfRendered,
+            "ssl" => SSLConfRendered
+        }
     ),
     Hocon.
 
@@ -144,6 +149,7 @@ hocon_config_template() ->
 bootstrap_hosts = \"{{ kafka_hosts_string }}\"
 enable = true
 authentication = {{{ authentication }}} 
+ssl = {{{ ssl }}}
 producer = {
     mqtt {
        topic = \"t/#\"
@@ -173,12 +179,65 @@ hocon_config_template_authentication(#{"kerberos_principal" := _}) ->
 }
 """.
 
+%% erlfmt-ignore
+hocon_config_template_ssl(Map) when map_size(Map) =:= 0 ->
+"""
+{
+    enable = false
+}
+""";
+hocon_config_template_ssl(_) ->
+"""
+{
+    enable = true
+    cacertfile = \"{{{cacertfile}}}\"
+    certfile = \"{{{certfile}}}\"
+    keyfile = \"{{{keyfile}}}\"
+}
+""".
+
 kafka_hosts_string() ->
     "kafka-1.emqx.net:9092,".
 
 kafka_hosts_string_sasl() ->
     "kafka-1.emqx.net:9093,".
 
+kafka_hosts_string_ssl() ->
+    "kafka-1.emqx.net:9094,".
+
+kafka_hosts_string_ssl_sasl() ->
+    "kafka-1.emqx.net:9095,".
+
+valid_ssl_settings() ->
+    #{
+        "cacertfile" => <<"/var/lib/secret/ca.crt">>,
+        "certfile" => <<"/var/lib/secret/client.crt">>,
+        "keyfile" => <<"/var/lib/secret/client.key">>
+    }.
+
+valid_sasl_plain_settings() ->
+    #{
+        "mechanism" => "plain",
+        "username" => "emqxuser",
+        "password" => "password"
+    }.
+
+valid_sasl_scram256_settings() ->
+    (valid_sasl_plain_settings())#{
+        "mechanism" => "scram_sha_256"
+    }.
+
+valid_sasl_scram512_settings() ->
+    (valid_sasl_plain_settings())#{
+        "mechanism" => "scram_sha_512"
+    }.
+
+valid_sasl_kerberos_settings() ->
+    #{
+        "kerberos_principal" => "rig@KDC.EMQX.NET",
+        "kerberos_keytab_file" => "/var/lib/secret/rig.keytab"
+    }.
+
 kafka_hosts() ->
     kpro:parse_endpoints(kafka_hosts_string()).
 

From 9aa7e826cbef5c2b8a30eb5afce9bd8de47253b2 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Sat, 17 Sep 2022 00:34:30 +0800
Subject: [PATCH 154/232] refactor(resource): fast resume resource worker if
 inflight msgs are ACKed

---
 .../test/emqx_bridge_mqtt_SUITE.erl           |   4 +-
 .../src/emqx_resource_manager.erl             |   2 +
 .../src/emqx_resource_worker.erl              | 145 +++++++++++-------
 .../test/emqx_connector_demo.erl              |   2 +
 .../test/emqx_resource_SUITE.erl              |  29 ++--
 5 files changed, 113 insertions(+), 69 deletions(-)

diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
index c89621d59..4ffeee71f 100644
--- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -570,9 +570,9 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
                 <<"success">> := 3,
                 <<"failed">> := 0,
                 <<"queuing">> := 0,
-                <<"retried">> := R
+                <<"retried">> := _
             }
-        } when R > 0,
+        },
         jsx:decode(BridgeStr2)
     ),
     %% also verify the 2 messages have been sent to the remote broker
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 3430b3964..8d61832e3 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -131,6 +131,8 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) ->
         [
             'matched',
             'retried',
+            'retried.success',
+            'retried.failed',
             'success',
             'failed',
             'dropped',
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 25032c29c..4f7ac63d8 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -56,9 +56,12 @@
 
 -define(Q_ITEM(REQUEST), {q_item, REQUEST}).
 
--define(QUERY(FROM, REQUEST), {query, FROM, REQUEST}).
--define(REPLY(FROM, REQUEST, RESULT), {reply, FROM, REQUEST, RESULT}).
--define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]).
+-define(QUERY(FROM, REQUEST, SENT), {query, FROM, REQUEST, SENT}).
+-define(REPLY(FROM, REQUEST, SENT, RESULT), {reply, FROM, REQUEST, SENT, RESULT}).
+-define(EXPAND(RESULT, BATCH), [
+    ?REPLY(FROM, REQUEST, SENT, RESULT)
+ || ?QUERY(FROM, REQUEST, SENT) <- BATCH
+]).
 
 -type id() :: binary().
 -type query() :: {query, from(), request()}.
@@ -89,16 +92,16 @@ async_query(Id, Request, Opts) ->
 %% simple query the resource without batching and queuing messages.
 -spec simple_sync_query(id(), request()) -> Result :: term().
 simple_sync_query(Id, Request) ->
-    Result = call_query(sync, Id, ?QUERY(self(), Request), #{}),
+    Result = call_query(sync, Id, ?QUERY(self(), Request, false), #{}),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
-    _ = handle_query_result(Id, Result, false),
+    _ = handle_query_result(Id, Result, false, false),
     Result.
 
 -spec simple_async_query(id(), request(), reply_fun()) -> Result :: term().
 simple_async_query(Id, Request, ReplyFun) ->
-    Result = call_query(async, Id, ?QUERY(ReplyFun, Request), #{}),
+    Result = call_query(async, Id, ?QUERY(ReplyFun, Request, false), #{}),
     ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
-    _ = handle_query_result(Id, Result, false),
+    _ = handle_query_result(Id, Result, false, false),
     Result.
 
 -spec block(pid() | atom()) -> ok.
@@ -156,7 +159,7 @@ running(cast, resume, _St) ->
     keep_state_and_data;
 running(cast, block, St) ->
     {next_state, blocked, St};
-running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{id := Id, queue := Q} = St) when
+running(cast, {block, [?QUERY(_, _, _) | _] = Batch}, #{id := Id, queue := Q} = St) when
     is_list(Batch)
 ->
     Q1 = maybe_append_queue(Id, Q, [?Q_ITEM(Query) || Query <- Batch]),
@@ -178,7 +181,7 @@ blocked(enter, _, #{resume_interval := ResumeT} = _St) ->
     {keep_state_and_data, {state_timeout, ResumeT, resume}};
 blocked(cast, block, _St) ->
     keep_state_and_data;
-blocked(cast, {block, [?QUERY(_, _) | _] = Batch}, #{id := Id, queue := Q} = St) when
+blocked(cast, {block, [?QUERY(_, _, _) | _] = Batch}, #{id := Id, queue := Q} = St) when
     is_list(Batch)
 ->
     Q1 = maybe_append_queue(Id, Q, [?Q_ITEM(Query) || Query <- Batch]),
@@ -189,13 +192,15 @@ blocked(state_timeout, resume, St) ->
     do_resume(St);
 blocked({call, From}, {query, Request, _Opts}, #{id := Id, queue := Q} = St) ->
     Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
-    _ = reply_caller(Id, ?REPLY(From, Request, Error)),
-    {keep_state, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(From, Request))])}};
+    _ = reply_caller(Id, ?REPLY(From, Request, false, Error)),
+    {keep_state, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(From, Request, false))])}};
 blocked(cast, {query, Request, Opts}, #{id := Id, queue := Q} = St) ->
     ReplayFun = maps:get(async_reply_fun, Opts, undefined),
     Error = ?RESOURCE_ERROR(blocked, "resource is blocked"),
-    _ = reply_caller(Id, ?REPLY(ReplayFun, Request, Error)),
-    {keep_state, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(ReplayFun, Request))])}}.
+    _ = reply_caller(Id, ?REPLY(ReplayFun, Request, false, Error)),
+    {keep_state, St#{
+        queue := maybe_append_queue(Id, Q, [?Q_ITEM(?QUERY(ReplayFun, Request, false))])
+    }}.
 
 terminate(_Reason, #{id := Id, index := Index}) ->
     gproc_pool:disconnect_worker(Id, {Id, Index}).
@@ -250,10 +255,11 @@ retry_first_from_queue(Q, Id, St) ->
             retry_first_sync(Id, FirstQuery, undefined, undefined, Q, St)
     end.
 
-retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = St0) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
-    Result = call_query(sync, Id, FirstQuery, #{}),
-    case handle_query_result(Id, Result, false) of
+retry_first_sync(
+    Id, ?QUERY(_, _, HasSent) = Query, Name, Ref, Q, #{resume_interval := ResumeT} = St0
+) ->
+    Result = call_query(sync, Id, Query, #{}),
+    case handle_query_result(Id, Result, HasSent, false) of
         %% Send failed because resource down
         true ->
             {keep_state, St0, {state_timeout, ResumeT, resume}};
@@ -279,7 +285,7 @@ drop_head(Id, Q) ->
     Q1.
 
 query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left, id := Id} = St0) ->
-    Acc1 = [?QUERY(From, Request) | Acc],
+    Acc1 = [?QUERY(From, Request, false) | Acc],
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'batching'),
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
@@ -290,10 +296,10 @@ query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id} = St)
     QueryOpts = #{
         inflight_name => maps:get(name, St)
     },
-    Result = call_query(configured, Id, ?QUERY(From, Request), QueryOpts),
-    case reply_caller(Id, ?REPLY(From, Request, Result)) of
+    Result = call_query(configured, Id, ?QUERY(From, Request, false), QueryOpts),
+    case reply_caller(Id, ?REPLY(From, Request, false, Result)) of
         true ->
-            Query = ?QUERY(From, Request),
+            Query = ?QUERY(From, Request, 1),
             {next_state, blocked, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(Query)])}};
         false ->
             {keep_state, St}
@@ -361,63 +367,65 @@ batch_reply_caller(Id, BatchResult, Batch) ->
 reply_caller(Id, Reply) ->
     reply_caller(Id, Reply, false).
 
-reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) ->
-    handle_query_result(Id, Result, BlockWorker);
-reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) ->
+reply_caller(Id, ?REPLY(undefined, _, HasSent, Result), BlockWorker) ->
+    handle_query_result(Id, Result, HasSent, BlockWorker);
+reply_caller(Id, ?REPLY({ReplyFun, Args}, _, HasSent, Result), BlockWorker) when
+    is_function(ReplyFun)
+->
     _ =
         case Result of
             {async_return, _} -> no_reply_for_now;
             _ -> apply(ReplyFun, Args ++ [Result])
         end,
-    handle_query_result(Id, Result, BlockWorker);
-reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) ->
+    handle_query_result(Id, Result, HasSent, BlockWorker);
+reply_caller(Id, ?REPLY(From, _, HasSent, Result), BlockWorker) ->
     gen_statem:reply(From, Result),
-    handle_query_result(Id, Result, BlockWorker).
+    handle_query_result(Id, Result, HasSent, BlockWorker).
 
-handle_query_result(Id, ?RESOURCE_ERROR_M(exception, Msg), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(exception, Msg), HasSent, BlockWorker) ->
     ?SLOG(error, #{msg => resource_exception, info => Msg}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
+    inc_sent_failed(Id, HasSent),
     BlockWorker;
-handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _) when
+handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasSent, _) when
     NotWorking == not_connected; NotWorking == blocked
 ->
     true;
-handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => resource_not_found, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_not_found'),
     BlockWorker;
-handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => resource_stopped, info => Msg}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_stopped'),
     BlockWorker;
-handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), BlockWorker) ->
+handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => other_resource_error, reason => Reason}),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.other'),
     BlockWorker;
-handle_query_result(Id, {error, {recoverable_error, Reason}}, _BlockWorker) ->
+handle_query_result(Id, {error, {recoverable_error, Reason}}, _HasSent, _BlockWorker) ->
     %% the message will be queued in replayq or inflight window,
     %% i.e. the counter 'queuing' or 'dropped' will increase, so we pretend that we have not
     %% sent this message.
     ?SLOG(warning, #{id => Id, msg => recoverable_error, reason => Reason}),
     true;
-handle_query_result(Id, {error, Reason}, BlockWorker) ->
+handle_query_result(Id, {error, Reason}, HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => send_error, reason => Reason}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
+    inc_sent_failed(Id, HasSent),
     BlockWorker;
-handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) ->
+handle_query_result(_Id, {async_return, inflight_full}, _HasSent, _BlockWorker) ->
     true;
-handle_query_result(Id, {async_return, {error, Msg}}, BlockWorker) ->
+handle_query_result(Id, {async_return, {error, Msg}}, HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => async_send_error, info => Msg}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
+    inc_sent_failed(Id, HasSent),
     BlockWorker;
-handle_query_result(_Id, {async_return, ok}, BlockWorker) ->
+handle_query_result(_Id, {async_return, ok}, _HasSent, BlockWorker) ->
     BlockWorker;
-handle_query_result(Id, Result, BlockWorker) ->
+handle_query_result(Id, Result, HasSent, BlockWorker) ->
     assert_ok_result(Result),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'success'),
+    inc_sent_success(Id, HasSent),
     BlockWorker.
 
 call_query(QM0, Id, Query, QueryOpts) ->
@@ -456,10 +464,10 @@ call_query(QM0, Id, Query, QueryOpts) ->
     end
 ).
 
-apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) ->
+apply_query_fun(sync, Mod, Id, ?QUERY(_, Request, _) = _Query, ResSt, _QueryOpts) ->
     ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}),
     ?APPLY_RESOURCE(call_query, Mod:on_query(Id, Request, ResSt), Request);
-apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
+apply_query_fun(async, Mod, Id, ?QUERY(_, Request, _) = Query, ResSt, QueryOpts) ->
     ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
     ?APPLY_RESOURCE(
@@ -478,11 +486,11 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) ->
         end,
         Request
     );
-apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) ->
+apply_query_fun(sync, Mod, Id, [?QUERY(_, _, _) | _] = Batch, ResSt, _QueryOpts) ->
     ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
-    Requests = [Request || ?QUERY(_From, Request) <- Batch],
+    Requests = [Request || ?QUERY(_From, Request, _) <- Batch],
     ?APPLY_RESOURCE(call_batch_query, Mod:on_batch_query(Id, Requests, ResSt), Batch);
-apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
+apply_query_fun(async, Mod, Id, [?QUERY(_, _, _) | _] = Batch, ResSt, QueryOpts) ->
     ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}),
     Name = maps:get(inflight_name, QueryOpts, undefined),
     ?APPLY_RESOURCE(
@@ -496,7 +504,7 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
                 ReplyFun = fun ?MODULE:batch_reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = {ReplyFun, [self(), Id, Name, Ref, Batch]},
-                Requests = [Request || ?QUERY(_From, Request) <- Batch],
+                Requests = [Request || ?QUERY(_From, Request, _) <- Batch],
                 ok = inflight_append(Name, Ref, Batch),
                 Result = Mod:on_batch_query_async(Id, Requests, Args, ResSt),
                 {async_return, Result}
@@ -504,18 +512,18 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) ->
         Batch
     ).
 
-reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request), Result) ->
+reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request, HasSent), Result) ->
     %% NOTE: 'inflight' is message count that sent async but no ACK received,
     %%        NOT the message number ququed in the inflight window.
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight', -1),
-    case reply_caller(Id, ?REPLY(From, Request, Result)) of
+    case reply_caller(Id, ?REPLY(From, Request, HasSent, Result)) of
         true ->
             %% we marked these messages are 'queuing' although they are actually
             %% keeped in inflight window, not replayq
             emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
             ?MODULE:block(Pid);
         false ->
-            inflight_drop(Name, Ref)
+            drop_inflight_and_resume(Pid, Name, Ref)
     end.
 
 batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
@@ -529,9 +537,19 @@ batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
             %% keeped in inflight window, not replayq
             emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', BatchLen),
             ?MODULE:block(Pid);
+        false ->
+            drop_inflight_and_resume(Pid, Name, Ref)
+    end.
+
+drop_inflight_and_resume(Pid, Name, Ref) ->
+    case inflight_is_full(Name) of
+        true ->
+            inflight_drop(Name, Ref),
+            ?MODULE:resume(Pid);
         false ->
             inflight_drop(Name, Ref)
     end.
+
 %%==============================================================================
 %% the inflight queue for async query
 -define(SIZE_REF, -1).
@@ -565,8 +583,14 @@ inflight_is_full(Name) ->
 
 inflight_append(undefined, _Ref, _Query) ->
     ok;
-inflight_append(Name, Ref, Query) ->
-    ets:insert(Name, {Ref, Query}),
+inflight_append(Name, Ref, [?QUERY(_, _, _) | _] = Batch) ->
+    ets:insert(Name, {Ref, [?QUERY(From, Req, true) || ?QUERY(From, Req, _) <- Batch]}),
+    ok;
+inflight_append(Name, Ref, ?QUERY(From, Req, _)) ->
+    ets:insert(Name, {Ref, ?QUERY(From, Req, true)}),
+    ok;
+inflight_append(Name, Ref, Data) ->
+    ets:insert(Name, {Ref, Data}),
     ok.
 
 inflight_drop(undefined, _) ->
@@ -576,6 +600,21 @@ inflight_drop(Name, Ref) ->
     ok.
 
 %%==============================================================================
+
+inc_sent_failed(Id, true) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried.failed');
+inc_sent_failed(Id, _HasSent) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed').
+
+inc_sent_success(Id, true) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'success'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried.success');
+inc_sent_success(Id, _HasSent) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'success').
+
 call_mode(sync, _) -> sync;
 call_mode(async, always_sync) -> sync;
 call_mode(async, async_if_possible) -> async.
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index a645af7d8..3b83cf7ed 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -170,9 +170,11 @@ counter_loop(#{counter := Num, status := Status} = State) ->
                 ct:pal("counter recv: ~p, buffered msgs: ~p", [resume, length(Msgs)]),
                 State#{status => running};
             {inc, N, ReplyFun} when Status == running ->
+                %ct:pal("async counter recv: ~p", [{inc, N}]),
                 apply_reply(ReplyFun, ok),
                 State#{counter => Num + N};
             {{FromPid, ReqRef}, {inc, N}} when Status == running ->
+                %ct:pal("sync counter recv: ~p", [{inc, N}]),
                 FromPid ! {ReqRef, ok},
                 State#{counter => Num + N};
             {{FromPid, ReqRef}, {inc, _N}} when Status == blocked ->
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index a9d274168..2446c8102 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -211,7 +211,7 @@ t_batch_query_counter(_) ->
         ?DEFAULT_RESOURCE_GROUP,
         ?TEST_RESOURCE,
         #{name => test_resource, register => true},
-        #{enable_batch => true}
+        #{enable_batch => true, query_mode => sync}
     ),
 
     ?check_trace(
@@ -220,7 +220,7 @@ t_batch_query_counter(_) ->
         fun(Result, Trace) ->
             ?assertMatch({ok, 0}, Result),
             QueryTrace = ?of_kind(call_batch_query, Trace),
-            ?assertMatch([#{batch := [{query, _, get_counter}]}], QueryTrace)
+            ?assertMatch([#{batch := [{query, _, get_counter, _}]}], QueryTrace)
         end
     ),
 
@@ -251,7 +251,7 @@ t_query_counter_async_query(_) ->
         fun(Trace) ->
             %% the callback_mode if 'emqx_connector_demo' is 'always_sync'.
             QueryTrace = ?of_kind(call_query, Trace),
-            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace)
         end
     ),
     %% wait for 1s to make sure all the aysnc query is sent to the resource.
@@ -264,7 +264,7 @@ t_query_counter_async_query(_) ->
             ?assertMatch({ok, 1000}, Result),
             %% the callback_mode if 'emqx_connector_demo' is 'always_sync'.
             QueryTrace = ?of_kind(call_query, Trace),
-            ?assertMatch([#{query := {query, _, get_counter}}], QueryTrace)
+            ?assertMatch([#{query := {query, _, get_counter, _}}], QueryTrace)
         end
     ),
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
@@ -292,7 +292,7 @@ t_query_counter_async_callback(_) ->
         inc_counter_in_parallel(1000, ReqOpts),
         fun(Trace) ->
             QueryTrace = ?of_kind(call_query_async, Trace),
-            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace)
         end
     ),
 
@@ -305,7 +305,7 @@ t_query_counter_async_callback(_) ->
         fun(Result, Trace) ->
             ?assertMatch({ok, 1000}, Result),
             QueryTrace = ?of_kind(call_query, Trace),
-            ?assertMatch([#{query := {query, _, get_counter}}], QueryTrace)
+            ?assertMatch([#{query := {query, _, get_counter, _}}], QueryTrace)
         end
     ),
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
@@ -341,7 +341,8 @@ t_query_counter_async_inflight(_) ->
             enable_batch => false,
             async_inflight_window => WindowSize,
             worker_pool_size => 1,
-            resume_interval => 300
+            resume_interval => 300,
+            enable_queue => false
         }
     ),
     ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)),
@@ -355,11 +356,11 @@ t_query_counter_async_inflight(_) ->
         inc_counter_in_parallel(WindowSize, ReqOpts),
         fun(Trace) ->
             QueryTrace = ?of_kind(call_query_async, Trace),
-            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace)
         end
     ),
 
-    %% this will block the resource_worker as the inflight windown is full now
+    %% this will block the resource_worker as the inflight window is full now
     ok = emqx_resource:query(?ID, {inc_counter, 1}),
     ?assertMatch(0, ets:info(Tab0, size)),
     %% sleep to make the resource_worker resume some times
@@ -387,7 +388,7 @@ t_query_counter_async_inflight(_) ->
         inc_counter_in_parallel(Num, ReqOpts),
         fun(Trace) ->
             QueryTrace = ?of_kind(call_query_async, Trace),
-            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace)
         end
     ),
     timer:sleep(1000),
@@ -401,7 +402,7 @@ t_query_counter_async_inflight(_) ->
         inc_counter_in_parallel(WindowSize, ReqOpts),
         fun(Trace) ->
             QueryTrace = ?of_kind(call_query_async, Trace),
-            ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace)
+            ?assertMatch([#{query := {query, _, {inc_counter, 1}, _}} | _], QueryTrace)
         end
     ),
 
@@ -415,13 +416,13 @@ t_query_counter_async_inflight(_) ->
 
     {ok, Counter} = emqx_resource:simple_sync_query(?ID, get_counter),
     ct:pal("get_counter: ~p, sent: ~p", [Counter, Sent]),
-    ?assert(Sent == Counter),
+    ?assert(Sent =< Counter),
 
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
     ct:pal("metrics: ~p", [C]),
     ?assertMatch(
-        #{matched := M, success := Ss, dropped := D} when
-            M == Ss + D,
+        #{matched := M, success := Ss, dropped := Dp, 'retried.success' := Rs} when
+            M == Ss + Dp - Rs,
         C
     ),
     ?assert(

From 03495d8d3614e5bc5309003a435315f92ddd41b2 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 19 Sep 2022 16:53:15 +0800
Subject: [PATCH 155/232] fix: influxdb server string support scheme prefix

---
 lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 2024ba8ce..3f86792ff 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -485,8 +485,12 @@ log_error_points(InstId, Errs) ->
 %% ===================================================================
 %% typereflt funcs
 
--spec to_server_raw(string()) ->
+-spec to_server_raw(string() | binary()) ->
     {string(), pos_integer()}.
+to_server_raw(<<"http://", Server/binary>>) ->
+    emqx_connector_schema_lib:parse_server(Server, ?INFLUXDB_HOST_OPTIONS);
+to_server_raw(<<"https://", Server/binary>>) ->
+    emqx_connector_schema_lib:parse_server(Server, ?INFLUXDB_HOST_OPTIONS);
 to_server_raw(Server) ->
     emqx_connector_schema_lib:parse_server(Server, ?INFLUXDB_HOST_OPTIONS).
 

From 67888816428bb6f6d72aed78e32486aa7010e9db Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 19 Sep 2022 17:40:32 +0800
Subject: [PATCH 156/232] fix: hide influxdb udp config

---
 apps/emqx_bridge/src/emqx_bridge.erl         | 4 ++--
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index d4d24ef3a..1c2dddeb1 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -52,8 +52,8 @@
     T == webhook;
     T == mysql;
     T == influxdb_api_v1;
-    T == influxdb_api_v2;
-    T == influxdb_udp
+    T == influxdb_api_v2
+    %% T == influxdb_udp
 ).
 
 load() ->
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 840b963cd..47925bb36 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -19,7 +19,7 @@ api_schemas(Method) ->
         ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"),
         ref(emqx_ee_bridge_mongodb, Method ++ "_single"),
         ref(emqx_ee_bridge_hstreamdb, Method),
-        ref(emqx_ee_bridge_influxdb, Method ++ "_udp"),
+        %% ref(emqx_ee_bridge_influxdb, Method ++ "_udp"),
         ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"),
         ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2")
     ].
@@ -83,5 +83,9 @@ fields(influxdb) ->
                 hoconsc:map(name, ref(emqx_ee_bridge_influxdb, Protocol)),
                 #{desc => <<"EMQX Enterprise Config">>}
             )}
-     || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2]
+     || Protocol <- [
+            %% influxdb_udp,
+            influxdb_api_v1,
+            influxdb_api_v2
+        ]
     ].

From 89711881f5a7c7919869b5ab4055dda85fa3d4f9 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 20 Sep 2022 15:02:27 +0800
Subject: [PATCH 157/232] chore: update emqtt to 1.7.0-rc.2

To avoid the MQTT bridge blocked once the inflight window fulled
see: https://github.com/emqx/emqtt/pull/171
---
 mix.exs      | 2 +-
 rebar.config | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index 63c444eec..1274eb329 100644
--- a/mix.exs
+++ b/mix.exs
@@ -59,7 +59,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true},
       {:replayq, "0.3.4", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
-      {:emqtt, github: "emqx/emqtt", tag: "1.7.0-rc.1", override: true},
+      {:emqtt, github: "emqx/emqtt", tag: "1.7.0-rc.2", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
diff --git a/rebar.config b/rebar.config
index e29c5f2c7..4a02104f0 100644
--- a/rebar.config
+++ b/rebar.config
@@ -61,7 +61,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
     , {replayq, "0.3.4"}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0-rc.1"}}}
+    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0-rc.2"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}

From b32563339032f1cc00a7fb61082e264bfa212ba7 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 19 Sep 2022 14:26:46 +0800
Subject: [PATCH 158/232] refactor(resource): resume from queue/inflight-window
 with async-sending and batching

---
 .../src/emqx_resource_worker.erl              | 167 +++++++++++-------
 1 file changed, 105 insertions(+), 62 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 4f7ac63d8..288edcf4f 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -208,14 +208,6 @@ terminate(_Reason, #{id := Id, index := Index}) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-queue_item_marshaller(?Q_ITEM(_) = I) ->
-    term_to_binary(I);
-queue_item_marshaller(Bin) when is_binary(Bin) ->
-    binary_to_term(Bin).
-
-estimate_size(QItem) ->
-    size(queue_item_marshaller(QItem)).
-
 %%==============================================================================
 -define(PICK(ID, KEY, EXPR),
     try gproc_pool:pick_worker(ID, KEY) of
@@ -237,26 +229,60 @@ pick_call(Id, Key, Query, Timeout) ->
 pick_cast(Id, Key, Query) ->
     ?PICK(Id, Key, gen_statem:cast(Pid, Query)).
 
-do_resume(#{queue := Q, id := Id, name := Name} = St) ->
+do_resume(#{id := Id, name := Name} = St) ->
     case inflight_get_first(Name) of
         empty ->
-            retry_first_from_queue(Q, Id, St);
+            retry_queue(St);
         {Ref, FirstQuery} ->
-            retry_first_sync(Id, FirstQuery, Name, Ref, undefined, St)
+            %% We retry msgs in inflight window sync, as if we send them
+            %% async, they will be appended to the end of inflight window again.
+            retry_inflight_sync(Id, Ref, FirstQuery, Name, St)
     end.
 
-retry_first_from_queue(undefined, _Id, St) ->
+retry_queue(#{queue := undefined} = St) ->
     {next_state, running, St};
-retry_first_from_queue(Q, Id, St) ->
-    case replayq:peek(Q) of
-        empty ->
+retry_queue(#{queue := Q, id := Id, enable_batch := false, resume_interval := ResumeT} = St) ->
+    case get_first_n_from_queue(Q, 1) of
+        [] ->
             {next_state, running, St};
-        ?Q_ITEM(FirstQuery) ->
-            retry_first_sync(Id, FirstQuery, undefined, undefined, Q, St)
+        [?QUERY(_, Request, HasSent) = Query] ->
+            QueryOpts = #{inflight_name => maps:get(name, St)},
+            Result = call_query(configured, Id, Query, QueryOpts),
+            case reply_caller(Id, ?REPLY(undefined, Request, HasSent, Result)) of
+                true ->
+                    {keep_state, St, {state_timeout, ResumeT, resume}};
+                false ->
+                    retry_queue(St#{queue := drop_head(Q, Id)})
+            end
+    end;
+retry_queue(
+    #{
+        queue := Q,
+        id := Id,
+        enable_batch := true,
+        batch_size := BatchSize,
+        resume_interval := ResumeT
+    } = St
+) ->
+    case get_first_n_from_queue(Q, BatchSize) of
+        [] ->
+            {next_state, running, St};
+        Batch0 ->
+            QueryOpts = #{inflight_name => maps:get(name, St)},
+            Result = call_query(configured, Id, Batch0, QueryOpts),
+            %% The caller has been replied with ?RESOURCE_ERROR(blocked, _) before saving into the queue,
+            %% we now change the 'from' field to 'undefined' so it will not reply the caller again.
+            Batch = [?QUERY(undefined, Request, HasSent) || ?QUERY(_, Request, HasSent) <- Batch0],
+            case batch_reply_caller(Id, Result, Batch) of
+                true ->
+                    {keep_state, St, {state_timeout, ResumeT, resume}};
+                false ->
+                    retry_queue(St#{queue := drop_first_n_from_queue(Q, length(Batch), Id)})
+            end
     end.
 
-retry_first_sync(
-    Id, ?QUERY(_, _, HasSent) = Query, Name, Ref, Q, #{resume_interval := ResumeT} = St0
+retry_inflight_sync(
+    Id, Ref, ?QUERY(_, _, HasSent) = Query, Name, #{resume_interval := ResumeT} = St0
 ) ->
     Result = call_query(sync, Id, Query, #{}),
     case handle_query_result(Id, Result, HasSent, false) of
@@ -265,25 +291,10 @@ retry_first_sync(
             {keep_state, St0, {state_timeout, ResumeT, resume}};
         %% Send ok or failed but the resource is working
         false ->
-            %% We Send 'resume' to the end of the mailbox to give the worker
-            %% a chance to process 'query' requests.
-            St =
-                case Q of
-                    undefined ->
-                        inflight_drop(Name, Ref),
-                        St0;
-                    _ ->
-                        St0#{queue => drop_head(Id, Q)}
-                end,
-            {keep_state, St, {state_timeout, 0, resume}}
+            inflight_drop(Name, Ref),
+            do_resume(St0)
     end.
 
-drop_head(Id, Q) ->
-    {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
-    ok = replayq:ack(Q1, AckRef),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -1),
-    Q1.
-
 query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left, id := Id} = St0) ->
     Acc1 = [?QUERY(From, Request, false) | Acc],
     emqx_metrics_worker:inc(?RES_METRICS, Id, 'batching'),
@@ -299,7 +310,7 @@ query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id} = St)
     Result = call_query(configured, Id, ?QUERY(From, Request, false), QueryOpts),
     case reply_caller(Id, ?REPLY(From, Request, false, Result)) of
         true ->
-            Query = ?QUERY(From, Request, 1),
+            Query = ?QUERY(From, Request, false),
             {next_state, blocked, St#{queue := maybe_append_queue(Id, Q, [?Q_ITEM(Query)])}};
         false ->
             {keep_state, St}
@@ -330,29 +341,6 @@ flush(
             {keep_state, St1}
     end.
 
-maybe_append_queue(Id, undefined, _Items) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
-    undefined;
-maybe_append_queue(Id, Q, Items) ->
-    Q2 =
-        case replayq:overflow(Q) of
-            Overflow when Overflow =< 0 ->
-                Q;
-            Overflow ->
-                PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
-                {Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts),
-                ok = replayq:ack(Q1, QAckRef),
-                Dropped = length(Items2),
-                emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -Dropped),
-                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
-                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
-                ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
-                Q1
-        end,
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
-    replayq:append(Q2, Items).
-
 batch_reply_caller(Id, BatchResult, Batch) ->
     lists:foldl(
         fun(Reply, BlockWorker) ->
@@ -550,12 +538,67 @@ drop_inflight_and_resume(Pid, Name, Ref) ->
             inflight_drop(Name, Ref)
     end.
 
+%%==============================================================================
+%% operations for queue
+queue_item_marshaller(?Q_ITEM(_) = I) ->
+    term_to_binary(I);
+queue_item_marshaller(Bin) when is_binary(Bin) ->
+    binary_to_term(Bin).
+
+estimate_size(QItem) ->
+    size(queue_item_marshaller(QItem)).
+
+maybe_append_queue(Id, undefined, _Items) ->
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
+    undefined;
+maybe_append_queue(Id, Q, Items) ->
+    Q2 =
+        case replayq:overflow(Q) of
+            Overflow when Overflow =< 0 ->
+                Q;
+            Overflow ->
+                PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
+                {Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts),
+                ok = replayq:ack(Q1, QAckRef),
+                Dropped = length(Items2),
+                emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -Dropped),
+                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
+                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
+                ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
+                Q1
+        end,
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
+    replayq:append(Q2, Items).
+
+get_first_n_from_queue(Q, N) ->
+    get_first_n_from_queue(Q, N, []).
+
+get_first_n_from_queue(_Q, 0, Acc) ->
+    lists:reverse(Acc);
+get_first_n_from_queue(Q, N, Acc) when N > 0 ->
+    case replayq:peek(Q) of
+        empty -> Acc;
+        ?Q_ITEM(Query) -> get_first_n_from_queue(Q, N - 1, [Query | Acc])
+    end.
+
+drop_first_n_from_queue(Q, 0, _Id) ->
+    Q;
+drop_first_n_from_queue(Q, N, Id) when N > 0 ->
+    drop_first_n_from_queue(drop_head(Q, Id), N - 1, Id).
+
+drop_head(Q, Id) ->
+    {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
+    ok = replayq:ack(Q1, AckRef),
+    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -1),
+    Q1.
+
 %%==============================================================================
 %% the inflight queue for async query
 -define(SIZE_REF, -1).
 inflight_new(Name, InfltWinSZ) ->
     _ = ets:new(Name, [named_table, ordered_set, public, {write_concurrency, true}]),
-    inflight_append(Name, ?SIZE_REF, {size, InfltWinSZ}),
+    inflight_append(Name, ?SIZE_REF, {max_size, InfltWinSZ}),
     ok.
 
 inflight_get_first(Name) ->
@@ -575,7 +618,7 @@ inflight_get_first(Name) ->
 inflight_is_full(undefined) ->
     false;
 inflight_is_full(Name) ->
-    [{_, {size, MaxSize}}] = ets:lookup(Name, ?SIZE_REF),
+    [{_, {max_size, MaxSize}}] = ets:lookup(Name, ?SIZE_REF),
     case ets:info(Name, size) of
         Size when Size > MaxSize -> true;
         _ -> false

From 48bb116905ec65b49f8c00ae9e18c0383f86ca11 Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Fri, 23 Sep 2022 13:50:12 +0800
Subject: [PATCH 159/232] chore: update dashboard vsn

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f98449081..9de423d10 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v1.0.8
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.2
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.4
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

From ba34326010e46f8a240a2bc301c3f6bd46800621 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 23 Sep 2022 08:58:51 +0200
Subject: [PATCH 160/232] ci(kafka): fix shellcheck errors

---
 .../kafka/generate-certs.sh                   | 33 ++++++++++---------
 .../kafka/run_add_scram_users.sh              |  2 ++
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/.ci/docker-compose-file/kafka/generate-certs.sh b/.ci/docker-compose-file/kafka/generate-certs.sh
index d0ae4a8d0..3f1c75550 100755
--- a/.ci/docker-compose-file/kafka/generate-certs.sh
+++ b/.ci/docker-compose-file/kafka/generate-certs.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/bash
 
 set -euo pipefail
 
@@ -17,29 +17,30 @@ cd /var/lib/secret/
 
 ls
 
-echo == Generate self-signed server and client certificates
-echo = generate CA
+echo '== Generate self-signed server and client certificates'
+echo '= generate CA'
 openssl req -new -x509 -keyout ca.key -out ca.crt -days $DAYS -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
 
-echo = generate server certificate request
-openssl req -newkey rsa:2048 -sha256 -keyout server.key -out server.csr -days $DAYS -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
+echo '= generate server certificate request'
+openssl req -newkey rsa:2048 -sha256 -keyout server.key -out server.csr -days "$DAYS" -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
 
-echo = sign server certificate
-openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days $DAYS -CAcreateserial
+echo '= sign server certificate'
+openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days "$DAYS" -CAcreateserial
 
-echo = generate client certificate request
-openssl req -newkey rsa:2048 -sha256 -keyout client.key -out client.csr -days $DAYS -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
+echo '= generate client certificate request'
+openssl req -newkey rsa:2048 -sha256 -keyout client.key -out client.csr -days "$DAYS" -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST"
 
-echo == sign client certificate
+echo '== sign client certificate'
 openssl x509 -req -CA ca.crt -CAkey ca.key -in client.csr -out client.crt -days $DAYS -CAserial ca.srl
 
-echo = Convert self-signed certificate to PKCS#12 format
-openssl pkcs12 -export -name $HOST -in server.crt -inkey server.key -out server.p12 -CAfile ca.crt -passout pass:$PASS
+echo '= Convert self-signed certificate to PKCS#12 format'
+openssl pkcs12 -export -name "$HOST" -in server.crt -inkey server.key -out server.p12 -CAfile ca.crt -passout pass:"$PASS"
 
-echo = Import PKCS#12 into a java keystore
+echo '= Import PKCS#12 into a java keystore'
 
-echo $PASS | keytool -importkeystore -destkeystore kafka.keystore.jks -srckeystore server.p12 -srcstoretype pkcs12 -alias $HOST -storepass $PASS
+echo $PASS | keytool -importkeystore -destkeystore kafka.keystore.jks -srckeystore server.p12 -srcstoretype pkcs12 -alias "$HOST" -storepass "$PASS"
 
-echo = Import CA into java truststore
 
-echo yes | keytool -keystore kafka.truststore.jks -alias CARoot -import -file ca.crt -storepass $PASS
+echo '= Import CA into java truststore'
+
+echo yes | keytool -keystore kafka.truststore.jks -alias CARoot -import -file ca.crt -storepass "$PASS"
diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
index 32f42a9e9..e997a310c 100755
--- a/.ci/docker-compose-file/kafka/run_add_scram_users.sh
+++ b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
@@ -31,8 +31,10 @@ TIMEOUT=60
 
 echo "+++++++ Wait until Kafka ports are up ++++++++"
 
+# shellcheck disable=SC2016
 timeout $TIMEOUT bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1
 
+# shellcheck disable=SC2016
 timeout $TIMEOUT bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT2
 
 echo "+++++++ Run config commands ++++++++"

From 516d60c7dad14f7c33cb4f8a990388286816d98d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 23 Sep 2022 09:00:17 +0200
Subject: [PATCH 161/232] build: fix deps consistency check

---
 apps/emqx/rebar.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 242f95fcb..8c7635d58 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -22,7 +22,7 @@
 %% This rebar.config is necessary because the app may be used as a
 %% `git_subdir` dependency in other projects.
 {deps, [
-    {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}},
+    {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}},
     {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}},
     {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},

From adc67b165b641862a88fc6b5d7090ce288985280 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 23 Sep 2022 10:07:22 +0200
Subject: [PATCH 162/232] test: test cases for Kafka bridge REST API

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      |   2 +-
 .../emqx_bridge_impl_kafka_producer_SUITE.erl | 321 +++++++++++++++++-
 2 files changed, 318 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index a353c9cf0..6bfa439d1 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -92,7 +92,7 @@ param_path_operation_cluster() ->
             #{
                 in => path,
                 required => true,
-                example => <<"start">>,
+                example => <<"restart">>,
                 desc => ?DESC("desc_param_path_operation_cluster")
             }
         )}.
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 9ca87d106..19ab05cc5 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -13,6 +13,34 @@
 
 -define(PRODUCER, emqx_bridge_impl_kafka).
 
+%%------------------------------------------------------------------------------
+%% Things for REST API tests
+%%------------------------------------------------------------------------------
+
+-import(
+    emqx_common_test_http,
+    [
+        request_api/3,
+        request_api/5,
+        get_http_data/1
+    ]
+).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include("emqx_dashboard.hrl").
+
+-define(CONTENT_TYPE, "application/x-www-form-urlencoded").
+
+-define(HOST, "http://127.0.0.1:18083").
+
+%% -define(API_VERSION, "v5").
+
+-define(BASE_PATH, "/api/v5").
+
+-define(APP_DASHBOARD, emqx_dashboard).
+-define(APP_MANAGEMENT, emqx_management).
+
 %%------------------------------------------------------------------------------
 %% CT boilerplate
 %%------------------------------------------------------------------------------
@@ -36,13 +64,89 @@ wait_until_kafka_is_up(Attempts) ->
     end.
 
 init_per_suite(Config) ->
-    {ok, _} = application:ensure_all_started(brod),
-    {ok, _} = application:ensure_all_started(wolff),
+    %% Need to unload emqx_authz. See emqx_machine_SUITE:init_per_suite for
+    %% more info.
+    application:unload(emqx_authz),
+    emqx_common_test_helpers:start_apps(
+        [emqx_conf, emqx_rule_engine, emqx_bridge, emqx_management, emqx_dashboard],
+        fun set_special_configs/1
+    ),
+    application:set_env(emqx_machine, applications, [
+        emqx_prometheus,
+        emqx_modules,
+        emqx_dashboard,
+        emqx_gateway,
+        emqx_statsd,
+        emqx_resource,
+        emqx_rule_engine,
+        emqx_bridge,
+        emqx_ee_bridge,
+        emqx_plugin_libs,
+        emqx_management,
+        emqx_retainer,
+        emqx_exhook,
+        emqx_authn,
+        emqx_authz,
+        emqx_plugin
+    ]),
+    {ok, _} = application:ensure_all_started(emqx_machine),
     wait_until_kafka_is_up(),
+    %% Wait until bridges API is up
+    (fun WaitUntilRestApiUp() ->
+        case show(http_get(["bridges"])) of
+            {ok, 200, _Res} ->
+                ok;
+            Val ->
+                ct:pal("REST API for bridges not up. Wait and try again. Response: ~p", [Val]),
+                timer:sleep(1000),
+                WaitUntilRestApiUp()
+        end
+    end)(),
     Config.
 
-end_per_suite(_) ->
+end_per_suite(Config) ->
+    emqx_common_test_helpers:stop_apps([
+        emqx_prometheus,
+        emqx_modules,
+        emqx_dashboard,
+        emqx_gateway,
+        emqx_statsd,
+        emqx_resource,
+        emqx_rule_engine,
+        emqx_bridge,
+        emqx_ee_bridge,
+        emqx_plugin_libs,
+        emqx_management,
+        emqx_retainer,
+        emqx_exhook,
+        emqx_authn,
+        emqx_authz,
+        emqx_plugin,
+        emqx_conf,
+        emqx_bridge,
+        emqx_management,
+        emqx_dashboard,
+        emqx_machine
+    ]),
+    mria:stop(),
+    Config.
+
+set_special_configs(emqx_management) ->
+    Listeners = #{http => #{port => 8081}},
+    Config = #{
+        listeners => Listeners,
+        applications => [#{id => "admin", secret => "public"}]
+    },
+    emqx_config:put([emqx_management], Config),
+    ok;
+set_special_configs(emqx_dashboard) ->
+    emqx_dashboard_api_test_helpers:set_default_config(),
+    ok;
+set_special_configs(_) ->
     ok.
+%%------------------------------------------------------------------------------
+%% Test cases for all combinations of SSL, no SSL and authentication types
+%%------------------------------------------------------------------------------
 
 t_publish_no_auth(_CtConfig) ->
     publish_with_and_without_ssl("none").
@@ -59,6 +163,160 @@ t_publish_sasl_scram512(_CtConfig) ->
 t_publish_sasl_kerberos(_CtConfig) ->
     publish_with_and_without_ssl(valid_sasl_kerberos_settings()).
 
+%%------------------------------------------------------------------------------
+%% Test cases for REST api
+%%------------------------------------------------------------------------------
+
+show(X) ->
+    % erlang:display('______________ SHOW ______________:'),
+    % erlang:display(X),
+    X.
+
+t_kafka_bridge_rest_api_plain_text(_CtConfig) ->
+    kafka_bridge_rest_api_all_auth_methods(false).
+
+t_kafka_bridge_rest_api_ssl(_CtConfig) ->
+    kafka_bridge_rest_api_all_auth_methods(true).
+
+kafka_bridge_rest_api_all_auth_methods(UseSSL) ->
+    NormalHostsString =
+        case UseSSL of
+            true -> kafka_hosts_string_ssl();
+            false -> kafka_hosts_string()
+        end,
+    kafka_bridge_rest_api_helper(#{
+        <<"bootstrap_hosts">> => NormalHostsString,
+        <<"authentication">> => <<"none">>
+    }),
+    SASLHostsString =
+        case UseSSL of
+            true -> kafka_hosts_string_ssl_sasl();
+            false -> kafka_hosts_string_sasl()
+        end,
+    BinifyMap = fun(Map) ->
+        maps:from_list([
+            {erlang:iolist_to_binary(K), erlang:iolist_to_binary(V)}
+         || {K, V} <- maps:to_list(Map)
+        ])
+    end,
+    SSLSettings =
+        case UseSSL of
+            true -> #{<<"ssl">> => BinifyMap(valid_ssl_settings())};
+            false -> #{}
+        end,
+    kafka_bridge_rest_api_helper(
+        maps:merge(
+            #{
+                <<"bootstrap_hosts">> => SASLHostsString,
+                <<"authentication">> => BinifyMap(valid_sasl_plain_settings())
+            },
+            SSLSettings
+        )
+    ),
+    kafka_bridge_rest_api_helper(
+        maps:merge(
+            #{
+                <<"bootstrap_hosts">> => SASLHostsString,
+                <<"authentication">> => BinifyMap(valid_sasl_scram256_settings())
+            },
+            SSLSettings
+        )
+    ),
+    kafka_bridge_rest_api_helper(
+        maps:merge(
+            #{
+                <<"bootstrap_hosts">> => SASLHostsString,
+                <<"authentication">> => BinifyMap(valid_sasl_scram512_settings())
+            },
+            SSLSettings
+        )
+    ),
+    kafka_bridge_rest_api_helper(
+        maps:merge(
+            #{
+                <<"bootstrap_hosts">> => SASLHostsString,
+                <<"authentication">> => BinifyMap(valid_sasl_kerberos_settings())
+            },
+            SSLSettings
+        )
+    ),
+    ok.
+
+kafka_bridge_rest_api_helper(Config) ->
+    UrlEscColon = "%3A",
+    BridgeIdUrlEnc = "kafka" ++ UrlEscColon ++ "my_kafka_bridge",
+    BridgesParts = ["bridges"],
+    BridgesPartsId = ["bridges", BridgeIdUrlEnc],
+    OpUrlFun = fun(OpName) -> ["bridges", BridgeIdUrlEnc, "operation", OpName] end,
+    BridgesPartsOpDisable = OpUrlFun("disable"),
+    BridgesPartsOpEnable = OpUrlFun("enable"),
+    BridgesPartsOpRestart = OpUrlFun("restart"),
+    BridgesPartsOpStop = OpUrlFun("stop"),
+    %% List bridges
+    MyKafkaBridgeExists = fun() ->
+        {ok, _Code, BridgesData} = show(http_get(BridgesParts)),
+        Bridges = show(json(BridgesData)),
+        lists:any(
+            fun
+                (#{<<"name">> := <<"my_kafka_bridge">>}) -> true;
+                (_) -> false
+            end,
+            Bridges
+        )
+    end,
+    %% Delete if my_kafka_bridge exists
+    case MyKafkaBridgeExists() of
+        true ->
+            %% Delete the bridge my_kafka_bridge
+            show(
+                '========================================== DELETE ========================================'
+            ),
+            {ok, 204, <<>>} = show(http_delete(BridgesPartsId));
+        false ->
+            ok
+    end,
+    false = MyKafkaBridgeExists(),
+    %% Create new Kafka bridge
+    CreateBodyTmp = #{
+        <<"type">> => <<"kafka">>,
+        <<"name">> => <<"my_kafka_bridge">>,
+        <<"bootstrap_hosts">> => maps:get(<<"bootstrap_hosts">>, Config),
+        <<"enable">> => true,
+        <<"authentication">> => maps:get(<<"authentication">>, Config),
+        <<"producer">> => #{
+            <<"mqtt">> => #{
+                topic => <<"t/#">>
+            },
+            <<"kafka">> => #{
+                <<"topic">> => <<"test-topic-one-partition">>
+            }
+        }
+    },
+    CreateBody =
+        case maps:is_key(<<"ssl">>, Config) of
+            true -> CreateBodyTmp#{<<"ssl">> => maps:get(<<"ssl">>, Config)};
+            false -> CreateBodyTmp
+        end,
+    {ok, 201, _Data} = show(http_post(BridgesParts, show(CreateBody))),
+    %% Check that the new bridge is in the list of bridges
+    true = MyKafkaBridgeExists(),
+    %% Perform operations
+    {ok, 200, _} = show(http_post(show(BridgesPartsOpDisable), #{})),
+    {ok, 200, _} = show(http_post(show(BridgesPartsOpDisable), #{})),
+    {ok, 200, _} = show(http_post(show(BridgesPartsOpEnable), #{})),
+    {ok, 200, _} = show(http_post(show(BridgesPartsOpEnable), #{})),
+    {ok, 200, _} = show(http_post(show(BridgesPartsOpStop), #{})),
+    {ok, 200, _} = show(http_post(show(BridgesPartsOpStop), #{})),
+    {ok, 200, _} = show(http_post(show(BridgesPartsOpRestart), #{})),
+    %% Cleanup
+    {ok, 204, _} = show(http_delete(BridgesPartsId)),
+    false = MyKafkaBridgeExists(),
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Helper functions
+%%------------------------------------------------------------------------------
+
 publish_with_and_without_ssl(AuthSettings) ->
     publish_helper(#{
         auth_settings => AuthSettings,
@@ -212,7 +470,8 @@ valid_ssl_settings() ->
     #{
         "cacertfile" => <<"/var/lib/secret/ca.crt">>,
         "certfile" => <<"/var/lib/secret/client.crt">>,
-        "keyfile" => <<"/var/lib/secret/client.key">>
+        "keyfile" => <<"/var/lib/secret/client.key">>,
+        "enable" => <<"true">>
     }.
 
 valid_sasl_plain_settings() ->
@@ -243,3 +502,57 @@ kafka_hosts() ->
 
 resolve_kafka_offset(Hosts, Topic, Partition) ->
     brod:resolve_offset(Hosts, Topic, Partition, latest).
+
+%%------------------------------------------------------------------------------
+%% Internal functions rest API helpers
+%%------------------------------------------------------------------------------
+
+bin(X) -> iolist_to_binary(X).
+
+random_num() ->
+    erlang:system_time(nanosecond).
+
+http_get(Parts) ->
+    request_api(get, api_path(Parts), auth_header_()).
+
+http_delete(Parts) ->
+    request_api(delete, api_path(Parts), auth_header_()).
+
+http_post(Parts, Body) ->
+    request_api(post, api_path(Parts), [], auth_header_(), Body).
+
+http_put(Parts, Body) ->
+    request_api(put, api_path(Parts), [], auth_header_(), Body).
+
+request_dashboard(Method, Url, Auth) ->
+    Request = {Url, [Auth]},
+    do_request_dashboard(Method, Request).
+request_dashboard(Method, Url, QueryParams, Auth) ->
+    Request = {Url ++ "?" ++ QueryParams, [Auth]},
+    do_request_dashboard(Method, Request).
+do_request_dashboard(Method, Request) ->
+    ct:pal("Method: ~p, Request: ~p", [Method, Request]),
+    case httpc:request(Method, Request, [], []) of
+        {error, socket_closed_remotely} ->
+            {error, socket_closed_remotely};
+        {ok, {{"HTTP/1.1", Code, _}, _Headers, Return}} when
+            Code >= 200 andalso Code =< 299
+        ->
+            {ok, Return};
+        {ok, {Reason, _, _}} ->
+            {error, Reason}
+    end.
+
+auth_header_() ->
+    auth_header_(<<"admin">>, <<"public">>).
+
+auth_header_(Username, Password) ->
+    {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
+    {"Authorization", "Bearer " ++ binary_to_list(Token)}.
+
+api_path(Parts) ->
+    ?HOST ++ filename:join([?BASE_PATH | Parts]).
+
+json(Data) ->
+    {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]),
+    Jsx.

From 9f3e38aeb03c29175325f2779c70407f72ecf80a Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 23 Sep 2022 10:37:30 +0200
Subject: [PATCH 163/232] test: fix error in script detected by spellcheck
 script

---
 .ci/docker-compose-file/kerberos/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/docker-compose-file/kerberos/run.sh b/.ci/docker-compose-file/kerberos/run.sh
index 85f172207..c9580073f 100755
--- a/.ci/docker-compose-file/kerberos/run.sh
+++ b/.ci/docker-compose-file/kerberos/run.sh
@@ -3,8 +3,8 @@
 
 echo "Remove old keytabs"
 
-rm -f /var/lib/secret/kafka.keytab 2>&1 > /dev/null
-rm -f /var/lib/secret/rig.keytab 2>&1 > /dev/null
+rm -f /var/lib/secret/kafka.keytab > /dev/null 2>&1
+rm -f /var/lib/secret/rig.keytab > /dev/null 2>&1
 
 echo "Create realm"
 

From c4f7d385b57a90c4eadd65135163332eb7d09c39 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 23 Sep 2022 11:47:12 +0200
Subject: [PATCH 164/232] test: fix true not allowed as docker compose env var
 value

---
 .ci/docker-compose-file/docker-compose-kafka.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index 3bb7748d5..d610316bd 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -48,7 +48,7 @@ services:
       KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
-      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
       KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks

From 7b601bf970328ef7bf7704754e8eeb039b8e0e4a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 23 Sep 2022 11:52:12 +0200
Subject: [PATCH 165/232] chore: delete bad parse_bridge function clause

---
 apps/emqx_bridge/src/emqx_bridge.erl                           | 2 --
 apps/emqx_bridge/src/emqx_bridge_resource.erl                  | 2 --
 .../test/emqx_bridge_impl_kafka_producer_SUITE.erl             | 3 ++-
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index ceca5ea7f..d4d24ef3a 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -334,8 +334,6 @@ get_matched_bridges(Topic) ->
         Bridges
     ).
 
-%% TODO: refactor to return bridge type, and bridge name directly
-%% so there is no need to parse the id back to type and name at where it is used
 get_matched_bridge_id(_BType, #{enable := false}, _Topic, _BName, Acc) ->
     Acc;
 get_matched_bridge_id(BType, #{local_topic := Filter}, Topic, BName, Acc) when
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index b6fd2e7be..2894ec461 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -71,8 +71,6 @@ bridge_id(BridgeType, BridgeName) ->
     Type = bin(BridgeType),
     <<Type/binary, ":", Name/binary>>.
 
-parse_bridge_id(<<"bridge:", BridgeId/binary>>) ->
-    parse_bridge_id(BridgeId);
 parse_bridge_id(BridgeId) ->
     case string:split(bin(BridgeId), ":", all) of
         [Type, Name] -> {binary_to_atom(Type, utf8), binary_to_atom(Name, utf8)};
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 19ab05cc5..90ddd6933 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -383,7 +383,8 @@ config(Args) ->
         #{atom_key => true}
     ),
     InstId = maps:get("instance_id", Args),
-    Parsed#{bridge_name => erlang:element(2, emqx_bridge_resource:parse_bridge_id(InstId))}.
+    <<"bridge:", BridgeId>> = InstId,
+    Parsed#{bridge_name => erlang:element(2, emqx_bridge_resource:parse_bridge_id(BridgeId))}.
 
 hocon_config(Args) ->
     AuthConf = maps:get("authentication", Args),

From d24441b204e3eb6d142019363257c928317cefeb Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Fri, 23 Sep 2022 18:32:39 +0800
Subject: [PATCH 166/232] fix(bridges): some issues found in code review

---
 apps/emqx_bridge/i18n/emqx_bridge_schema.conf |  4 ++--
 apps/emqx_bridge/src/emqx_bridge.erl          | 14 +++++++-------
 apps/emqx_bridge/src/emqx_bridge_api.erl      |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
index 08fe9c299..c465ef242 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
@@ -173,8 +173,8 @@ emqx_bridge_schema {
                   }
      metric_retried {
                    desc {
-                         en: """Times of retried from the queue or the inflight window."""
-                         zh: """从队列或者飞行窗口里重试的次数。"""
+                         en: """Times of retried."""
+                         zh: """重试的次数。"""
                         }
                    label: {
                            en: "Retried"
diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index 1c2dddeb1..3aff30859 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -249,19 +249,19 @@ remove(BridgeType, BridgeName) ->
     ).
 
 check_deps_and_remove(BridgeType, BridgeName, RemoveDeps) ->
-    Id = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
+    BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
     %% NOTE: This violates the design: Rule depends on data-bridge but not vice versa.
-    case emqx_rule_engine:get_rule_ids_by_action(Id) of
+    case emqx_rule_engine:get_rule_ids_by_action(BridgeId) of
         [] ->
             remove(BridgeType, BridgeName);
-        Rules when RemoveDeps =:= false ->
-            {error, {rules_deps_on_this_bridge, Rules}};
-        Rules when RemoveDeps =:= true ->
+        RuleIds when RemoveDeps =:= false ->
+            {error, {rules_deps_on_this_bridge, RuleIds}};
+        RuleIds when RemoveDeps =:= true ->
             lists:foreach(
                 fun(R) ->
-                    emqx_rule_engine:ensure_action_removed(R, Id)
+                    emqx_rule_engine:ensure_action_removed(R, BridgeId)
                 end,
-                Rules
+                RuleIds
             ),
             remove(BridgeType, BridgeName)
     end.
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 57652a0fc..d32115fff 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -331,7 +331,7 @@ schema("/bridges/:id") ->
             responses => #{
                 204 => <<"Bridge deleted">>,
                 400 => error_schema(['INVALID_ID'], "Update bridge failed"),
-                403 => error_schema('FORBIDDEN_REQUEST', "forbidden operation"),
+                403 => error_schema('FORBIDDEN_REQUEST', "Forbidden operation"),
                 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable")
             }
         }

From 8e514680d8fac1c3fc5eab7899a5f677ff891c30 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 23 Sep 2022 13:54:11 +0200
Subject: [PATCH 167/232] test: fix bad binary pattern

---
 .../test/emqx_bridge_impl_kafka_producer_SUITE.erl              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 90ddd6933..5e9dbfc73 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -383,7 +383,7 @@ config(Args) ->
         #{atom_key => true}
     ),
     InstId = maps:get("instance_id", Args),
-    <<"bridge:", BridgeId>> = InstId,
+    <<"bridge:", BridgeId/binary>> = InstId,
     Parsed#{bridge_name => erlang:element(2, emqx_bridge_resource:parse_bridge_id(BridgeId))}.
 
 hocon_config(Args) ->

From a3c88b40a02fc4ae7caa9f8cd825231fb1502e20 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 23 Sep 2022 14:33:41 +0200
Subject: [PATCH 168/232] test: changes to make Kafka container run in GitHub
 action

---
 .ci/docker-compose-file/docker-compose-kafka.yaml             | 4 ++++
 .../test/emqx_bridge_impl_kafka_producer_SUITE.erl            | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index d610316bd..d58f51146 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -33,10 +33,14 @@ services:
     ports:
       - "9092:9092"
       - "9093:9093"
+      - "9094:9094"
+      - "9095:9095"
     container_name: kafka-1.emqx.net
     hostname: kafka-1.emqx.net
     depends_on:
       - "kdc"
+      - "zookeeper"
+      - "ssl_cert_gen"
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 5e9dbfc73..fb929e692 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -51,7 +51,7 @@ all() ->
 wait_until_kafka_is_up() ->
     wait_until_kafka_is_up(0).
 
-wait_until_kafka_is_up(90) ->
+wait_until_kafka_is_up(300) ->
     ct:fail("Kafka is not up even though we have waited for a while");
 wait_until_kafka_is_up(Attempts) ->
     KafkaTopic = "test-topic-one-partition",

From ac37c5d58aca781f800023d60098f8e30f5347ed Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 23 Sep 2022 15:02:01 +0200
Subject: [PATCH 169/232] test: github actions debug printouts

---
 .ci/docker-compose-file/kafka/run_add_scram_users.sh | 3 +++
 scripts/ct/run.sh                                    | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
index e997a310c..4b51fee0d 100755
--- a/.ci/docker-compose-file/kafka/run_add_scram_users.sh
+++ b/.ci/docker-compose-file/kafka/run_add_scram_users.sh
@@ -44,3 +44,6 @@ kafka-configs.sh --bootstrap-server localhost:9092 --alter --add-config 'SCRAM-S
 echo "+++++++ Wait until Kafka ports are down ++++++++"
 
 bash -c 'while printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1
+
+echo "+++++++ Kafka ports are down ++++++++"
+
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 45d32767c..99af6d098 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -145,6 +145,12 @@ if [ "$ONLY_UP" = 'yes' ]; then
     exit 0
 fi
 
+sleep 10
+
+echo "DOCKER COMPOSE LOGS kafka_1"
+
+docker-compose $F_OPTIONS logs kafka_1
+
 if [ "$ATTACH" = 'yes' ]; then
     docker exec -it "$ERLANG_CONTAINER" bash
 elif [ "$CONSOLE" = 'yes' ]; then

From 5ec4b0a6ca975c4ff8f71a523efcc386aefd961d Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 23 Sep 2022 15:58:22 +0200
Subject: [PATCH 170/232] fix: fix entrypoint in docker compose for Kafka
 bridge test

---
 .ci/docker-compose-file/docker-compose-kafka.yaml | 1 +
 scripts/ct/run.sh                                 | 6 ------
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index d58f51146..ba0161293 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -15,6 +15,7 @@ services:
     volumes:
       - emqx-shared-secret:/var/lib/secret
       - ./kafka/generate-certs.sh:/bin/generate-certs.sh
+    entrypoint: /bin/sh
     command: /bin/generate-certs.sh
   kdc:
     hostname: kdc.emqx.net
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 99af6d098..45d32767c 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -145,12 +145,6 @@ if [ "$ONLY_UP" = 'yes' ]; then
     exit 0
 fi
 
-sleep 10
-
-echo "DOCKER COMPOSE LOGS kafka_1"
-
-docker-compose $F_OPTIONS logs kafka_1
-
 if [ "$ATTACH" = 'yes' ]; then
     docker exec -it "$ERLANG_CONTAINER" bash
 elif [ "$CONSOLE" = 'yes' ]; then

From 9fbd7b6e172d67978e8e7a584b005fbf81cb3b2d Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 26 Sep 2022 09:42:32 +0800
Subject: [PATCH 171/232] chore: bump `influxdb-client-erl` version

---
 mix.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index 8a764bccd..1f9de7951 100644
--- a/mix.exs
+++ b/mix.exs
@@ -129,7 +129,7 @@ defmodule EMQXUmbrella.MixProject do
   defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
     [
       {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
-      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.3", override: true},
+      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.4", override: true},
       {:wolff, github: "kafka4beam/wolff", tag: "1.6.4"},
       {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.0", override: true},
       {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"},

From dfe14be8b229fca2e3c76b25ea94ba6f81dee5ea Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 26 Sep 2022 18:23:28 +0800
Subject: [PATCH 172/232] chore: bump app vsns

---
 apps/emqx_bridge/src/emqx_bridge.app.src               | 2 +-
 apps/emqx_connector/src/emqx_connector.app.src         | 2 +-
 apps/emqx_dashboard/src/emqx_dashboard.app.src         | 2 +-
 apps/emqx_management/src/emqx_management.app.src       | 2 +-
 apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src     | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src       | 2 +-
 lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src           | 2 +-
 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src
index 8e1c60c29..7890853e4 100644
--- a/apps/emqx_bridge/src/emqx_bridge.app.src
+++ b/apps/emqx_bridge/src/emqx_bridge.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "An OTP application"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [
diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src
index 0734d47b8..06da66398 100644
--- a/apps/emqx_connector/src/emqx_connector.app.src
+++ b/apps/emqx_connector/src/emqx_connector.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "An OTP application"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index adf974243..9d5f85c7b 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.5"},
+    {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},
diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index d9f09f00e..5f8b30bf4 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.5"},
+    {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
index 10df22d97..bcdcfe420 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_plugin_libs, [
     {description, "EMQX Plugin utility libs"},
-    {vsn, "4.3.3"},
+    {vsn, "4.3.4"},
     {modules, []},
     {applications, [kernel, stdlib]},
     {env, []}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index 97c884fe9..0ede2a6a5 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -1,5 +1,5 @@
 {application, emqx_ee_bridge, [
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
index 324e7e308..771fdcb27 100644
--- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
+++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_conf, [
     {description, "EMQX Enterprise Edition configuration schema"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index c1b86d20b..b2aed3624 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,5 +1,5 @@
 {application, emqx_ee_connector, [
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [
         kernel,

From 19e3f5919e23c36fb293f187d8f28a75b52809af Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Mon, 26 Sep 2022 19:54:01 +0800
Subject: [PATCH 173/232] chore: release e5.0.0-beta.4

---
 apps/emqx/include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index e5311ef12..75cae1638 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -35,7 +35,7 @@
 -define(EMQX_RELEASE_CE, "5.0.8").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.0.0-beta.3").
+-define(EMQX_RELEASE_EE, "5.0.0-beta.4").
 
 %% the HTTP API version
 -define(EMQX_API_VERSION, "5.0").

From 57270fb8fc67ac56845313ccba4bc0fde1b7f7aa Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Wed, 28 Sep 2022 11:54:25 +0200
Subject: [PATCH 174/232] feat: add support for counters and gauges to the
 Kafka Bridge

This commit adds support for counters and gauges to the Kafka Brige.
The Kafka bridge uses [Wolff](https://github.com/kafka4beam/wolff) for
the  Kafka connection. Wolff does its own batching and does not use the
batching functionality in `emqx_resource_worker` that is used by other
bridge types. Therefore, the counter events have to be generated by
Wolff. We have added
[telemetry](https://github.com/beam-telemetry/telemetry) events to Wolff
that we hook into to change counters and gauges for the Kafka bridge. The
counter called `matched` does not depend on specific functionality of
any bridge type so the updates of this counter is moved higher up in the
call chain then previously so that it also gets updated for Kafka
bridges.
---
 .../src/emqx_resource_metrics.erl             | 197 ++++++++++++++++++
 .../src/emqx_resource_worker.erl              |  67 +++---
 lib-ee/emqx_ee_bridge/rebar.config            |   2 +-
 .../emqx_ee_bridge/src/emqx_ee_bridge.app.src |   3 +-
 .../kafka/emqx_bridge_impl_kafka_producer.erl |  99 ++++++++-
 .../emqx_bridge_impl_kafka_producer_SUITE.erl |  71 +++++--
 mix.exs                                       |   4 +-
 7 files changed, 389 insertions(+), 54 deletions(-)
 create mode 100644 apps/emqx_resource/src/emqx_resource_metrics.erl

diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl
new file mode 100644
index 000000000..4fe3d3182
--- /dev/null
+++ b/apps/emqx_resource/src/emqx_resource_metrics.erl
@@ -0,0 +1,197 @@
+-module(emqx_resource_metrics).
+
+-export([
+    batching_change/2,
+    batching_get/1,
+    inflight_change/2,
+    inflight_get/1,
+    queuing_change/2,
+    queuing_get/1,
+    dropped_inc/1,
+    dropped_inc/2,
+    dropped_get/1,
+    dropped_other_inc/1,
+    dropped_other_inc/2,
+    dropped_other_get/1,
+    dropped_queue_full_inc/1,
+    dropped_queue_full_inc/2,
+    dropped_queue_full_get/1,
+    dropped_queue_not_enabled_inc/1,
+    dropped_queue_not_enabled_inc/2,
+    dropped_queue_not_enabled_get/1,
+    dropped_resource_not_found_inc/1,
+    dropped_resource_not_found_inc/2,
+    dropped_resource_not_found_get/1,
+    dropped_resource_stopped_inc/1,
+    dropped_resource_stopped_inc/2,
+    dropped_resource_stopped_get/1,
+    failed_inc/1,
+    failed_inc/2,
+    failed_get/1,
+    matched_inc/1,
+    matched_inc/2,
+    matched_get/1,
+    retried_inc/1,
+    retried_inc/2,
+    retried_get/1,
+    retried_failed_inc/1,
+    retried_failed_inc/2,
+    retried_failed_get/1,
+    retried_success_inc/1,
+    retried_success_inc/2,
+    retried_success_get/1,
+    success_inc/1,
+    success_inc/2,
+    success_get/1
+]).
+
+-define(RES_METRICS, resource_metrics).
+
+%% Gauges (value can go both up and down):
+%% --------------------------------------
+
+%% @doc Count of messages that are currently accumulated in memory waiting for
+%% being sent in one batch
+batching_change(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'batching', Val).
+
+batching_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'batching').
+
+%% @doc Count of messages that are currently queuing. [Gauge]
+queuing_change(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'queuing', Val).
+
+queuing_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'queuing').
+
+%% @doc Count of messages that were sent asynchronously but ACKs are not
+%% received. [Gauge]
+inflight_change(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'inflight', Val).
+
+inflight_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'inflight').
+
+%% Counters (value can only got up):
+%% --------------------------------------
+
+%% @doc Count of messages dropped
+dropped_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped').
+
+dropped_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val).
+
+dropped_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped').
+
+%% @doc Count of messages dropped due to other reasons
+dropped_other_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.other').
+
+dropped_other_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.other', Val).
+
+dropped_other_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.other').
+
+%% @doc Count of messages dropped because the queue was full
+dropped_queue_full_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_full').
+
+dropped_queue_full_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_full', Val).
+
+dropped_queue_full_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.queue_full').
+
+%% @doc Count of messages dropped because the queue was not enabled
+dropped_queue_not_enabled_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_not_enabled').
+
+dropped_queue_not_enabled_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_not_enabled', Val).
+
+dropped_queue_not_enabled_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.queue_not_enabled').
+
+%% @doc Count of messages dropped because the resource was not found
+dropped_resource_not_found_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_not_found').
+
+dropped_resource_not_found_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_not_found', Val).
+
+dropped_resource_not_found_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.resource_not_found').
+
+%% @doc Count of messages dropped because the resource was stopped
+dropped_resource_stopped_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_stopped').
+
+dropped_resource_stopped_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_stopped', Val).
+
+dropped_resource_stopped_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.resource_stopped').
+
+%% @doc Count of how many times this bridge has been matched and queried
+matched_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'matched').
+
+matched_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'matched', Val).
+
+matched_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'matched').
+
+%% @doc The number of times message sends have been retried
+retried_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried').
+
+retried_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried', Val).
+
+retried_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'retried').
+
+%% @doc Count of message sends that have failed
+failed_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed').
+
+failed_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed', Val).
+
+failed_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'failed').
+
+%%% @doc Count of message sends that have failed after having been retried
+retried_failed_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.failed').
+
+retried_failed_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.failed', Val).
+
+retried_failed_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'retried.failed').
+
+%% @doc Count messages that were sucessfully sent after at least one retry
+retried_success_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.success').
+
+retried_success_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.success', Val).
+
+retried_success_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'retried.success').
+
+%% @doc Count of messages that have been sent successfully
+success_inc(ID) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'success').
+
+success_inc(ID, Val) ->
+    emqx_metrics_worker:inc(?RES_METRICS, ID, 'success', Val).
+
+success_get(ID) ->
+    emqx_metrics_worker:get(?RES_METRICS, ID, 'success').
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 288edcf4f..75a63d427 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -80,27 +80,23 @@ start_link(Id, Index, Opts) ->
 sync_query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
     Timeout = maps:get(timeout, Opts, infinity),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     pick_call(Id, PickKey, {query, Request, Opts}, Timeout).
 
 -spec async_query(id(), request(), query_opts()) -> Result :: term().
 async_query(Id, Request, Opts) ->
     PickKey = maps:get(pick_key, Opts, self()),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     pick_cast(Id, PickKey, {query, Request, Opts}).
 
 %% simple query the resource without batching and queuing messages.
 -spec simple_sync_query(id(), request()) -> Result :: term().
 simple_sync_query(Id, Request) ->
     Result = call_query(sync, Id, ?QUERY(self(), Request, false), #{}),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     _ = handle_query_result(Id, Result, false, false),
     Result.
 
 -spec simple_async_query(id(), request(), reply_fun()) -> Result :: term().
 simple_async_query(Id, Request, ReplyFun) ->
     Result = call_query(async, Id, ?QUERY(ReplyFun, Request, false), #{}),
-    ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'matched'),
     _ = handle_query_result(Id, Result, false, false),
     Result.
 
@@ -134,7 +130,7 @@ init({Id, Index, Opts}) ->
             false ->
                 undefined
         end,
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', queue_count(Queue)),
+    emqx_resource_metrics:queuing_change(Id, queue_count(Queue)),
     InfltWinSZ = maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT),
     ok = inflight_new(Name, InfltWinSZ),
     HCItvl = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL),
@@ -297,7 +293,7 @@ retry_inflight_sync(
 
 query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left, id := Id} = St0) ->
     Acc1 = [?QUERY(From, Request, false) | Acc],
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batching'),
+    emqx_resource_metrics:batching_change(Id, 1),
     St = St0#{acc := Acc1, acc_left := Left - 1},
     case Left =< 1 of
         true -> flush(St);
@@ -330,7 +326,7 @@ flush(
     QueryOpts = #{
         inflight_name => maps:get(name, St)
     },
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'batching', -length(Batch)),
+    emqx_resource_metrics:batching_change(Id, -length(Batch)),
     Result = call_query(configured, Id, Batch, QueryOpts),
     St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}),
     case batch_reply_caller(Id, Result, Batch) of
@@ -380,18 +376,18 @@ handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasSent, _) when
     true;
 handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => resource_not_found, info => Msg}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_not_found'),
+    emqx_resource_metrics:dropped_inc(Id),
+    emqx_resource_metrics:dropped_resource_not_found_inc(Id),
     BlockWorker;
 handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => resource_stopped, info => Msg}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.resource_stopped'),
+    emqx_resource_metrics:dropped_inc(Id),
+    emqx_resource_metrics:dropped_resource_stopped_inc(Id),
     BlockWorker;
 handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => other_resource_error, reason => Reason}),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.other'),
+    emqx_resource_metrics:dropped_inc(Id),
+    emqx_resource_metrics:dropped_other_inc(Id),
     BlockWorker;
 handle_query_result(Id, {error, {recoverable_error, Reason}}, _HasSent, _BlockWorker) ->
     %% the message will be queued in replayq or inflight window,
@@ -425,6 +421,7 @@ call_query(QM0, Id, Query, QueryOpts) ->
                     _ -> QM0
                 end,
             CM = maps:get(callback_mode, Data),
+            emqx_resource_metrics:matched_inc(Id),
             apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt, QueryOpts);
         {ok, _Group, #{status := stopped}} ->
             ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
@@ -464,7 +461,7 @@ apply_query_fun(async, Mod, Id, ?QUERY(_, Request, _) = Query, ResSt, QueryOpts)
             true ->
                 {async_return, inflight_full};
             false ->
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight'),
+                ok = emqx_resource_metrics:inflight_change(Id, 1),
                 ReplyFun = fun ?MODULE:reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = [self(), Id, Name, Ref, Query],
@@ -488,7 +485,7 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _, _) | _] = Batch, ResSt, QueryOpts)
                 {async_return, inflight_full};
             false ->
                 BatchLen = length(Batch),
-                ok = emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight', BatchLen),
+                ok = emqx_resource_metrics:inflight_change(Id, BatchLen),
                 ReplyFun = fun ?MODULE:batch_reply_after_query/6,
                 Ref = make_message_ref(),
                 Args = {ReplyFun, [self(), Id, Name, Ref, Batch]},
@@ -503,12 +500,12 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _, _) | _] = Batch, ResSt, QueryOpts)
 reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request, HasSent), Result) ->
     %% NOTE: 'inflight' is message count that sent async but no ACK received,
     %%        NOT the message number ququed in the inflight window.
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight', -1),
+    emqx_resource_metrics:inflight_change(Id, -1),
     case reply_caller(Id, ?REPLY(From, Request, HasSent, Result)) of
         true ->
             %% we marked these messages are 'queuing' although they are actually
             %% keeped in inflight window, not replayq
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
+            emqx_resource_metrics:queuing_change(Id, 1),
             ?MODULE:block(Pid);
         false ->
             drop_inflight_and_resume(Pid, Name, Ref)
@@ -518,12 +515,12 @@ batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) ->
     %% NOTE: 'inflight' is message count that sent async but no ACK received,
     %%        NOT the message number ququed in the inflight window.
     BatchLen = length(Batch),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'inflight', -BatchLen),
+    emqx_resource_metrics:inflight_change(Id, -BatchLen),
     case batch_reply_caller(Id, Result, Batch) of
         true ->
             %% we marked these messages are 'queuing' although they are actually
-            %% keeped in inflight window, not replayq
-            emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', BatchLen),
+            %% kept in inflight window, not replayq
+            emqx_resource_metrics:queuing_change(Id, BatchLen),
             ?MODULE:block(Pid);
         false ->
             drop_inflight_and_resume(Pid, Name, Ref)
@@ -549,8 +546,8 @@ estimate_size(QItem) ->
     size(queue_item_marshaller(QItem)).
 
 maybe_append_queue(Id, undefined, _Items) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
+    emqx_resource_metrics:dropped_inc(Id),
+    emqx_resource_metrics:dropped_queue_not_enabled_inc(Id),
     undefined;
 maybe_append_queue(Id, Q, Items) ->
     Q2 =
@@ -562,13 +559,13 @@ maybe_append_queue(Id, Q, Items) ->
                 {Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts),
                 ok = replayq:ack(Q1, QAckRef),
                 Dropped = length(Items2),
-                emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -Dropped),
-                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
-                emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
+                emqx_resource_metrics:queuing_change(Id, -Dropped),
+                emqx_resource_metrics:dropped_inc(Id),
+                emqx_resource_metrics:dropped_queue_full_inc(Id),
                 ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
                 Q1
         end,
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing'),
+    emqx_resource_metrics:queuing_change(Id, 1),
     replayq:append(Q2, Items).
 
 get_first_n_from_queue(Q, N) ->
@@ -590,7 +587,7 @@ drop_first_n_from_queue(Q, N, Id) when N > 0 ->
 drop_head(Q, Id) ->
     {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}),
     ok = replayq:ack(Q1, AckRef),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'queuing', -1),
+    emqx_resource_metrics:queuing_change(Id, -1),
     Q1.
 
 %%==============================================================================
@@ -645,18 +642,18 @@ inflight_drop(Name, Ref) ->
 %%==============================================================================
 
 inc_sent_failed(Id, true) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried.failed');
+    emqx_resource_metrics:failed_inc(Id),
+    emqx_resource_metrics:retried_inc(Id),
+    emqx_resource_metrics:retried_failed_inc(Id);
 inc_sent_failed(Id, _HasSent) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'failed').
+    emqx_resource_metrics:failed_inc(Id).
 
 inc_sent_success(Id, true) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'success'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried'),
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'retried.success');
+    emqx_resource_metrics:success_inc(Id),
+    emqx_resource_metrics:retried_inc(Id),
+    emqx_resource_metrics:retried_success_inc(Id);
 inc_sent_success(Id, _HasSent) ->
-    emqx_metrics_worker:inc(?RES_METRICS, Id, 'success').
+    emqx_resource_metrics:success_inc(Id).
 
 call_mode(sync, _) -> sync;
 call_mode(async, always_sync) -> sync;
diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config
index 8c79e7274..f281a9b85 100644
--- a/lib-ee/emqx_ee_bridge/rebar.config
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@@ -1,6 +1,6 @@
 {erl_opts, [debug_info]}.
 {deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}
-       , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.6.4"}}}
+       , {wolff, {git, "https://github.com/kjellwinblad/wolff.git", {branch, "kjell/add_counters_support_ok"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
        , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.4"}}}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index 0ede2a6a5..7759ef2a2 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -4,7 +4,8 @@
     {applications, [
         kernel,
         stdlib,
-        emqx_ee_connector
+        emqx_ee_connector,
+        telemetry
     ]},
     {env, []},
     {modules, []},
diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
index ce82dbe2d..5e8635d44 100644
--- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -12,7 +12,10 @@
     on_get_status/2
 ]).
 
--export([on_kafka_ack/3]).
+-export([
+    on_kafka_ack/3,
+    handle_telemetry_event/4
+]).
 
 -include_lib("emqx/include/logger.hrl").
 
@@ -30,6 +33,7 @@ on_start(InstId, Config) ->
         authentication := Auth,
         ssl := SSL
     } = Config,
+    maybe_install_wolff_telemetry_handlers(),
     %% it's a bug if producer config is not found
     %% the caller should not try to start a producer if
     %% there is no producer config
@@ -222,6 +226,7 @@ producers_config(BridgeName, ClientId, Input) ->
             disk -> {false, replayq_dir(ClientId)};
             hybrid -> {true, replayq_dir(ClientId)}
         end,
+    BridgeNameBin = erlang:atom_to_binary(BridgeName),
     #{
         name => make_producer_name(BridgeName),
         partitioner => PartitionStrategy,
@@ -234,7 +239,9 @@ producers_config(BridgeName, ClientId, Input) ->
         required_acks => RequiredAcks,
         max_batch_bytes => MaxBatchBytes,
         max_send_ahead => MaxInflight - 1,
-        compression => Compression
+        compression => Compression,
+        telemetry_meta_data =>
+            #{bridge_id => <<<<"bridge:kafka:">>/binary, BridgeNameBin/binary>>}
     }.
 
 replayq_dir(ClientId) ->
@@ -268,3 +275,91 @@ get_required(Field, Config, Throw) ->
     Value = maps:get(Field, Config, none),
     Value =:= none andalso throw(Throw),
     Value.
+
+handle_telemetry_event(
+    [wolff, dropped],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:dropped_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, dropped_queue_full],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:dropped_queue_full_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, queuing],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:queuing_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, retried],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:retried_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, failed],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:failed_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, inflight],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:inflight_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, retried_failed],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:retried_failed_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, retried_success],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:retried_success_inc(ID, Val);
+handle_telemetry_event(
+    [wolff, success],
+    #{counter_inc := Val},
+    #{bridge_id := ID},
+    _
+) when is_integer(Val) ->
+    emqx_resource_metrics:success_inc(ID, Val);
+handle_telemetry_event(_EventId, _Metrics, _MetaData, _Config) ->
+    %% Event that we do not handle
+    ok.
+
+maybe_install_wolff_telemetry_handlers() ->
+    %% Attach event handlers for Kafka telemetry events. If a handler with the
+    %% handler id already exists, the attach_many function does nothing
+    telemetry:attach_many(
+        %% unique handler id
+        <<"emqx-bridge-kafka-producer-telemetry-handler">>,
+        [
+            [wolff, dropped],
+            [wolff, dropped_queue_full],
+            [wolff, queuing],
+            [wolff, retried],
+            [wolff, failed],
+            [wolff, inflight],
+            [wolff, retried_failed],
+            [wolff, retried_success],
+            [wolff, success]
+        ],
+        fun emqx_bridge_impl_kafka_producer:handle_telemetry_event/4,
+        []
+    ).
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index fb929e692..152862f6b 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -184,10 +184,6 @@ kafka_bridge_rest_api_all_auth_methods(UseSSL) ->
             true -> kafka_hosts_string_ssl();
             false -> kafka_hosts_string()
         end,
-    kafka_bridge_rest_api_helper(#{
-        <<"bootstrap_hosts">> => NormalHostsString,
-        <<"authentication">> => <<"none">>
-    }),
     SASLHostsString =
         case UseSSL of
             true -> kafka_hosts_string_ssl_sasl();
@@ -204,6 +200,15 @@ kafka_bridge_rest_api_all_auth_methods(UseSSL) ->
             true -> #{<<"ssl">> => BinifyMap(valid_ssl_settings())};
             false -> #{}
         end,
+    kafka_bridge_rest_api_helper(
+        maps:merge(
+            #{
+                <<"bootstrap_hosts">> => NormalHostsString,
+                <<"authentication">> => <<"none">>
+            },
+            SSLSettings
+        )
+    ),
     kafka_bridge_rest_api_helper(
         maps:merge(
             #{
@@ -243,10 +248,20 @@ kafka_bridge_rest_api_all_auth_methods(UseSSL) ->
     ok.
 
 kafka_bridge_rest_api_helper(Config) ->
+    BridgeType = "kafka",
+    BridgeName = "my_kafka_bridge",
+    BridgeID = emqx_bridge_resource:bridge_id(
+        erlang:list_to_binary(BridgeType),
+        erlang:list_to_binary(BridgeName)
+    ),
+    ResourceId = emqx_bridge_resource:resource_id(
+        erlang:list_to_binary(BridgeType),
+        erlang:list_to_binary(BridgeName)
+    ),
     UrlEscColon = "%3A",
-    BridgeIdUrlEnc = "kafka" ++ UrlEscColon ++ "my_kafka_bridge",
+    BridgeIdUrlEnc = BridgeType ++ UrlEscColon ++ BridgeName,
     BridgesParts = ["bridges"],
-    BridgesPartsId = ["bridges", BridgeIdUrlEnc],
+    BridgesPartsIdDeleteAlsoActions = ["bridges", BridgeIdUrlEnc ++ "?also_delete_dep_actions"],
     OpUrlFun = fun(OpName) -> ["bridges", BridgeIdUrlEnc, "operation", OpName] end,
     BridgesPartsOpDisable = OpUrlFun("disable"),
     BridgesPartsOpEnable = OpUrlFun("enable"),
@@ -268,15 +283,13 @@ kafka_bridge_rest_api_helper(Config) ->
     case MyKafkaBridgeExists() of
         true ->
             %% Delete the bridge my_kafka_bridge
-            show(
-                '========================================== DELETE ========================================'
-            ),
-            {ok, 204, <<>>} = show(http_delete(BridgesPartsId));
+            {ok, 204, <<>>} = show(http_delete(BridgesPartsIdDeleteAlsoActions));
         false ->
             ok
     end,
     false = MyKafkaBridgeExists(),
     %% Create new Kafka bridge
+    KafkaTopic = "test-topic-one-partition",
     CreateBodyTmp = #{
         <<"type">> => <<"kafka">>,
         <<"name">> => <<"my_kafka_bridge">>,
@@ -288,7 +301,7 @@ kafka_bridge_rest_api_helper(Config) ->
                 topic => <<"t/#">>
             },
             <<"kafka">> => #{
-                <<"topic">> => <<"test-topic-one-partition">>
+                <<"topic">> => erlang:list_to_binary(KafkaTopic)
             }
         }
     },
@@ -300,6 +313,34 @@ kafka_bridge_rest_api_helper(Config) ->
     {ok, 201, _Data} = show(http_post(BridgesParts, show(CreateBody))),
     %% Check that the new bridge is in the list of bridges
     true = MyKafkaBridgeExists(),
+    %% Create a rule that uses the bridge
+    {ok, 201, _Rule} = http_post(
+        ["rules"],
+        #{
+            <<"name">> => <<"kafka_bridge_rest_api_helper_rule">>,
+            <<"enable">> => true,
+            <<"actions">> => [BridgeID],
+            <<"sql">> => <<"SELECT * from \"kafka_bridge_topic/#\"">>
+        }
+    ),
+    %% Get offset before sending message
+    {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0),
+    %% Send message to topic and check that it got forwarded to Kafka
+    Body = <<"message from EMQX">>,
+    emqx:publish(emqx_message:make(<<"kafka_bridge_topic/1">>, Body)),
+    %% Give Kafka some time to get message
+    timer:sleep(100),
+    %% Check that Kafka got message
+    BrodOut = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset),
+    {ok, {_, [KafkaMsg]}} = show(BrodOut),
+    Body = KafkaMsg#kafka_message.value,
+    %% Check crucial counters and gauges
+    1 = emqx_resource_metrics:matched_get(ResourceId),
+    1 = emqx_resource_metrics:success_get(ResourceId),
+    0 = emqx_resource_metrics:dropped_get(ResourceId),
+    0 = emqx_resource_metrics:failed_get(ResourceId),
+    0 = emqx_resource_metrics:inflight_get(ResourceId),
+    0 = emqx_resource_metrics:queuing_get(ResourceId),
     %% Perform operations
     {ok, 200, _} = show(http_post(show(BridgesPartsOpDisable), #{})),
     {ok, 200, _} = show(http_post(show(BridgesPartsOpDisable), #{})),
@@ -309,7 +350,7 @@ kafka_bridge_rest_api_helper(Config) ->
     {ok, 200, _} = show(http_post(show(BridgesPartsOpStop), #{})),
     {ok, 200, _} = show(http_post(show(BridgesPartsOpRestart), #{})),
     %% Cleanup
-    {ok, 204, _} = show(http_delete(BridgesPartsId)),
+    {ok, 204, _} = show(http_delete(BridgesPartsIdDeleteAlsoActions)),
     false = MyKafkaBridgeExists(),
     ok.
 
@@ -325,7 +366,8 @@ publish_with_and_without_ssl(AuthSettings) ->
     publish_helper(#{
         auth_settings => AuthSettings,
         ssl_settings => valid_ssl_settings()
-    }).
+    }),
+    ok.
 
 publish_helper(#{
     auth_settings := AuthSettings,
@@ -345,6 +387,7 @@ publish_helper(#{
     Hash = erlang:phash2([HostsString, AuthSettings, SSLSettings]),
     Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash),
     InstId = emqx_bridge_resource:resource_id("kafka", Name),
+    BridgeId = emqx_bridge_resource:bridge_id("kafka", Name),
     KafkaTopic = "test-topic-one-partition",
     Conf = config(#{
         "authentication" => AuthSettings,
@@ -353,6 +396,7 @@ publish_helper(#{
         "instance_id" => InstId,
         "ssl" => SSLSettings
     }),
+    emqx_bridge_resource:create(kafka, erlang:list_to_atom(Name), Conf, #{}),
     %% To make sure we get unique value
     timer:sleep(1),
     Time = erlang:monotonic_time(),
@@ -371,6 +415,7 @@ publish_helper(#{
     {ok, {_, [KafkaMsg]}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset),
     ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg),
     ok = ?PRODUCER:on_stop(InstId, State),
+    ok = emqx_bridge_resource:remove(BridgeId),
     ok.
 
 config(Args) ->
diff --git a/mix.exs b/mix.exs
index 69538142e..f9941912a 100644
--- a/mix.exs
+++ b/mix.exs
@@ -130,7 +130,7 @@ defmodule EMQXUmbrella.MixProject do
     [
       {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
       {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.4", override: true},
-      {:wolff, github: "kafka4beam/wolff", tag: "1.6.4"},
+      {:wolff, github: "kjellwinblad/wolff", branch: "kjell/add_counters_support_ok"},
       {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.0", override: true},
       {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"},
       {:brod, github: "kafka4beam/brod", tag: "3.16.4"},
@@ -516,7 +516,7 @@ defmodule EMQXUmbrella.MixProject do
     |> Path.join("RELEASES")
     |> File.open!([:write, :utf8], fn handle ->
       IO.puts(handle, "%% coding: utf-8")
-      :io.format(handle, '~tp.~n', [release_entry])
+      :io.format(handle, ~c"~tp.~n", [release_entry])
     end)
 
     release

From 98500313eb91fdc30765b8d0122601698c6cb9e2 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 10 Oct 2022 16:55:25 -0300
Subject: [PATCH 175/232] fix(kafka): some fixes for kafka producer

- MQTT topic should be a binary
- use correct gauge functions from `wolff_metrics`.
- don't double increment success counter for kafka action
- adds a few more metrics assertions
---
 .../src/emqx_ee_bridge_kafka.erl              |  2 +-
 .../kafka/emqx_bridge_impl_kafka_producer.erl | 46 +++++++++----------
 .../emqx_bridge_impl_kafka_producer_SUITE.erl | 39 +++++++++++++---
 3 files changed, 55 insertions(+), 32 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
index ac5177f6e..2540b987c 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
@@ -145,7 +145,7 @@ fields(producer_opts) ->
             })}
     ];
 fields(producer_mqtt_opts) ->
-    [{topic, mk(string(), #{desc => ?DESC(mqtt_topic)})}];
+    [{topic, mk(binary(), #{desc => ?DESC(mqtt_topic)})}];
 fields(producer_kafka_opts) ->
     [
         {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})},
diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
index 5e8635d44..09712c29d 100644
--- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -33,7 +33,7 @@ on_start(InstId, Config) ->
         authentication := Auth,
         ssl := SSL
     } = Config,
-    maybe_install_wolff_telemetry_handlers(),
+    _ = maybe_install_wolff_telemetry_handlers(),
     %% it's a bug if producer config is not found
     %% the caller should not try to start a producer if
     %% there is no producer config
@@ -226,7 +226,9 @@ producers_config(BridgeName, ClientId, Input) ->
             disk -> {false, replayq_dir(ClientId)};
             hybrid -> {true, replayq_dir(ClientId)}
         end,
-    BridgeNameBin = erlang:atom_to_binary(BridgeName),
+    %% TODO: change this once we add kafka source
+    BridgeType = kafka,
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName),
     #{
         name => make_producer_name(BridgeName),
         partitioner => PartitionStrategy,
@@ -240,8 +242,7 @@ producers_config(BridgeName, ClientId, Input) ->
         max_batch_bytes => MaxBatchBytes,
         max_send_ahead => MaxInflight - 1,
         compression => Compression,
-        telemetry_meta_data =>
-            #{bridge_id => <<<<"bridge:kafka:">>/binary, BridgeNameBin/binary>>}
+        telemetry_meta_data => #{bridge_id => ResourceID}
     }.
 
 replayq_dir(ClientId) ->
@@ -280,66 +281,59 @@ handle_telemetry_event(
     [wolff, dropped],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
     emqx_resource_metrics:dropped_inc(ID, Val);
 handle_telemetry_event(
     [wolff, dropped_queue_full],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
     emqx_resource_metrics:dropped_queue_full_inc(ID, Val);
 handle_telemetry_event(
     [wolff, queuing],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
-    emqx_resource_metrics:queuing_inc(ID, Val);
+    emqx_resource_metrics:queuing_change(ID, Val);
 handle_telemetry_event(
     [wolff, retried],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
     emqx_resource_metrics:retried_inc(ID, Val);
 handle_telemetry_event(
     [wolff, failed],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
     emqx_resource_metrics:failed_inc(ID, Val);
 handle_telemetry_event(
     [wolff, inflight],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
-    emqx_resource_metrics:inflight_inc(ID, Val);
+    emqx_resource_metrics:inflight_change(ID, Val);
 handle_telemetry_event(
     [wolff, retried_failed],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
     emqx_resource_metrics:retried_failed_inc(ID, Val);
 handle_telemetry_event(
     [wolff, retried_success],
     #{counter_inc := Val},
     #{bridge_id := ID},
-    _
+    _HandlerConfig
 ) when is_integer(Val) ->
     emqx_resource_metrics:retried_success_inc(ID, Val);
-handle_telemetry_event(
-    [wolff, success],
-    #{counter_inc := Val},
-    #{bridge_id := ID},
-    _
-) when is_integer(Val) ->
-    emqx_resource_metrics:success_inc(ID, Val);
-handle_telemetry_event(_EventId, _Metrics, _MetaData, _Config) ->
+handle_telemetry_event(_EventId, _Metrics, _MetaData, _HandlerConfig) ->
     %% Event that we do not handle
     ok.
 
@@ -349,6 +343,11 @@ maybe_install_wolff_telemetry_handlers() ->
     telemetry:attach_many(
         %% unique handler id
         <<"emqx-bridge-kafka-producer-telemetry-handler">>,
+        %% Note: we don't handle `[wolff, success]' because,
+        %% currently, we already increment the success counter for
+        %% this resource at `emqx_rule_runtime:handle_action' when
+        %% the response is `ok' and we would double increment it
+        %% here.
         [
             [wolff, dropped],
             [wolff, dropped_queue_full],
@@ -357,8 +356,7 @@ maybe_install_wolff_telemetry_handlers() ->
             [wolff, failed],
             [wolff, inflight],
             [wolff, retried_failed],
-            [wolff, retried_success],
-            [wolff, success]
+            [wolff, retried_success]
         ],
         fun emqx_bridge_impl_kafka_producer:handle_telemetry_event/4,
         []
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 152862f6b..2eef1170d 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -323,6 +323,22 @@ kafka_bridge_rest_api_helper(Config) ->
             <<"sql">> => <<"SELECT * from \"kafka_bridge_topic/#\"">>
         }
     ),
+    %% counters should be empty before
+    ?assertEqual(0, emqx_resource_metrics:matched_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:success_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:failed_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:inflight_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:batching_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:queuing_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_other_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_queue_full_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_queue_not_enabled_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_resource_not_found_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_resource_stopped_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:retried_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:retried_failed_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:retried_success_get(ResourceId)),
     %% Get offset before sending message
     {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0),
     %% Send message to topic and check that it got forwarded to Kafka
@@ -335,12 +351,21 @@ kafka_bridge_rest_api_helper(Config) ->
     {ok, {_, [KafkaMsg]}} = show(BrodOut),
     Body = KafkaMsg#kafka_message.value,
     %% Check crucial counters and gauges
-    1 = emqx_resource_metrics:matched_get(ResourceId),
-    1 = emqx_resource_metrics:success_get(ResourceId),
-    0 = emqx_resource_metrics:dropped_get(ResourceId),
-    0 = emqx_resource_metrics:failed_get(ResourceId),
-    0 = emqx_resource_metrics:inflight_get(ResourceId),
-    0 = emqx_resource_metrics:queuing_get(ResourceId),
+    ?assertEqual(1, emqx_resource_metrics:matched_get(ResourceId)),
+    ?assertEqual(1, emqx_resource_metrics:success_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:failed_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:inflight_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:batching_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:queuing_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_other_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_queue_full_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_queue_not_enabled_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_resource_not_found_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:dropped_resource_stopped_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:retried_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:retried_failed_get(ResourceId)),
+    ?assertEqual(0, emqx_resource_metrics:retried_success_get(ResourceId)),
     %% Perform operations
     {ok, 200, _} = show(http_post(show(BridgesPartsOpDisable), #{})),
     {ok, 200, _} = show(http_post(show(BridgesPartsOpDisable), #{})),
@@ -452,7 +477,7 @@ hocon_config_template() ->
 """
 bootstrap_hosts = \"{{ kafka_hosts_string }}\"
 enable = true
-authentication = {{{ authentication }}} 
+authentication = {{{ authentication }}}
 ssl = {{{ ssl }}}
 producer = {
     mqtt {

From cf361546f827c99b5e15260d4696a90418b4cce8 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 10 Oct 2022 17:38:11 -0300
Subject: [PATCH 176/232] feat: update emqx dashboard version -> e1.0.1-beta.5

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 654cec168..3dd11aec3 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v1.0.9
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.4
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.5
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

From 4475289ce40a626501265745370a4c5242279afd Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 11 Oct 2022 09:47:40 -0300
Subject: [PATCH 177/232] feat: use upstream newly tagged 1.7.0 wolff

---
 lib-ee/emqx_ee_bridge/rebar.config | 2 +-
 mix.exs                            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config
index f281a9b85..bfd1c957e 100644
--- a/lib-ee/emqx_ee_bridge/rebar.config
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@@ -1,6 +1,6 @@
 {erl_opts, [debug_info]}.
 {deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}
-       , {wolff, {git, "https://github.com/kjellwinblad/wolff.git", {branch, "kjell/add_counters_support_ok"}}}
+       , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
        , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.4"}}}
diff --git a/mix.exs b/mix.exs
index f9941912a..ad6a7d7b0 100644
--- a/mix.exs
+++ b/mix.exs
@@ -130,7 +130,7 @@ defmodule EMQXUmbrella.MixProject do
     [
       {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
       {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.4", override: true},
-      {:wolff, github: "kjellwinblad/wolff", branch: "kjell/add_counters_support_ok"},
+      {:wolff, github: "kafka4beam/wolff", tag: "1.7.0"},
       {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.0", override: true},
       {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"},
       {:brod, github: "kafka4beam/brod", tag: "3.16.4"},

From 357e5919cec9c4d8a456c438c9d05b8efff71e4c Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 11 Oct 2022 09:51:16 -0300
Subject: [PATCH 178/232] chore: add copyright disclaimer

---
 apps/emqx_resource/src/emqx_resource_metrics.erl | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl
index 4fe3d3182..96d955db0 100644
--- a/apps/emqx_resource/src/emqx_resource_metrics.erl
+++ b/apps/emqx_resource/src/emqx_resource_metrics.erl
@@ -1,3 +1,19 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
 -module(emqx_resource_metrics).
 
 -export([

From f0ff32c031dd3d27ae0f2ff8f8bd8bd65fa1f8d1 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 11 Oct 2022 17:37:59 -0300
Subject: [PATCH 179/232] test: fix tests after counter changes

---
 .../test/emqx_bridge_mqtt_SUITE.erl           |  3 ++-
 .../test/emqx_connector_demo.erl              | 24 ++++++++++++++++---
 .../test/emqx_resource_SUITE.erl              | 12 ++++++++--
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
index 4ffeee71f..819556d81 100644
--- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -540,6 +540,7 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
 
     %% stop the listener 1883 to make the bridge disconnected
     ok = emqx_listeners:stop_listener('tcp:default'),
+    ct:sleep(1500),
 
     %% PUBLISH 2 messages to the 'local' broker, the message should
     emqx:publish(emqx_message:make(LocalTopic, Payload)),
@@ -551,7 +552,7 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
         #{
             <<"status">> := Status,
             <<"metrics">> := #{
-                <<"matched">> := 3, <<"success">> := 1, <<"failed">> := 0, <<"queuing">> := 2
+                <<"matched">> := 1, <<"success">> := 1, <<"failed">> := 0, <<"queuing">> := 2
             }
         } when Status == <<"connected">> orelse Status == <<"connecting">>,
         jsx:decode(BridgeStr1)
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index 3b83cf7ed..105bcad77 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -100,6 +100,15 @@ on_query(_InstId, {inc_counter, N}, #{pid := Pid}) ->
     after 1000 ->
         {error, timeout}
     end;
+on_query(_InstId, get_incorrect_status_count, #{pid := Pid}) ->
+    ReqRef = make_ref(),
+    From = {self(), ReqRef},
+    Pid ! {From, get_incorrect_status_count},
+    receive
+        {ReqRef, Count} -> {ok, Count}
+    after 1000 ->
+        {error, timeout}
+    end;
 on_query(_InstId, get_counter, #{pid := Pid}) ->
     ReqRef = make_ref(),
     From = {self(), ReqRef},
@@ -157,9 +166,15 @@ spawn_counter_process(Name, Register) ->
     Pid.
 
 counter_loop() ->
-    counter_loop(#{counter => 0, status => running}).
+    counter_loop(#{counter => 0, status => running, incorrect_status_count => 0}).
 
-counter_loop(#{counter := Num, status := Status} = State) ->
+counter_loop(
+    #{
+        counter := Num,
+        status := Status,
+        incorrect_status_count := IncorrectCount
+    } = State
+) ->
     NewState =
         receive
             block ->
@@ -179,10 +194,13 @@ counter_loop(#{counter := Num, status := Status} = State) ->
                 State#{counter => Num + N};
             {{FromPid, ReqRef}, {inc, _N}} when Status == blocked ->
                 FromPid ! {ReqRef, incorrect_status},
-                State;
+                State#{incorrect_status_count := IncorrectCount + 1};
             {get, ReplyFun} ->
                 apply_reply(ReplyFun, Num),
                 State;
+            {{FromPid, ReqRef}, get_incorrect_status_count} ->
+                FromPid ! {ReqRef, IncorrectCount},
+                State;
             {{FromPid, ReqRef}, get} ->
                 FromPid ! {ReqRef, Num},
                 State
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 2446c8102..672e01896 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -420,10 +420,18 @@ t_query_counter_async_inflight(_) ->
 
     {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID),
     ct:pal("metrics: ~p", [C]),
+    {ok, IncorrectStatusCount} = emqx_resource:simple_sync_query(?ID, get_incorrect_status_count),
+    %% The `simple_sync_query' we just did also increases the matched
+    %% count, hence the + 1.
+    ExtraSimpleCallCount = IncorrectStatusCount + 1,
     ?assertMatch(
         #{matched := M, success := Ss, dropped := Dp, 'retried.success' := Rs} when
-            M == Ss + Dp - Rs,
-        C
+            M == Ss + Dp - Rs + ExtraSimpleCallCount,
+        C,
+        #{
+            metrics => C,
+            extra_simple_call_count => ExtraSimpleCallCount
+        }
     ),
     ?assert(
         lists:all(

From 24eda247ae58838f47b33f30bc4c0c12c8e16370 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 13 Oct 2022 09:27:24 -0300
Subject: [PATCH 180/232] chore: pin `telemetry` version

---
 mix.exs      | 3 ++-
 rebar.config | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index ad6a7d7b0..5750598ae 100644
--- a/mix.exs
+++ b/mix.exs
@@ -63,6 +63,7 @@ defmodule EMQXUmbrella.MixProject do
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
+      {:telemetry, "1.1.0"},
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
@@ -516,7 +517,7 @@ defmodule EMQXUmbrella.MixProject do
     |> Path.join("RELEASES")
     |> File.open!([:write, :utf8], fn handle ->
       IO.puts(handle, "%% coding: utf-8")
-      :io.format(handle, ~c"~tp.~n", [release_entry])
+      :io.format(handle, '~tp.~n', [release_entry])
     end)
 
     release
diff --git a/rebar.config b/rebar.config
index 769fe6e78..505c475cc 100644
--- a/rebar.config
+++ b/rebar.config
@@ -71,6 +71,7 @@
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}
+    , {telemetry, "1.1.0"}
     ]}.
 
 {xref_ignores,

From 1ad3b5df17c44279bb97289fdf1671c73acd726a Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 13 Oct 2022 09:58:57 -0300
Subject: [PATCH 181/232] fix: uninstall telemetry handler on resource stop,
 use unique id

---
 .../kafka/emqx_bridge_impl_kafka_producer.erl | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
index 09712c29d..0a03f582f 100644
--- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -33,7 +33,7 @@ on_start(InstId, Config) ->
         authentication := Auth,
         ssl := SSL
     } = Config,
-    _ = maybe_install_wolff_telemetry_handlers(),
+    _ = maybe_install_wolff_telemetry_handlers(InstId),
     %% it's a bug if producer config is not found
     %% the caller should not try to start a producer if
     %% there is no producer config
@@ -89,7 +89,7 @@ on_start(InstId, Config) ->
             throw(failed_to_start_kafka_producer)
     end.
 
-on_stop(_InstId, #{client_id := ClientID, producers := Producers}) ->
+on_stop(InstanceID, #{client_id := ClientID, producers := Producers}) ->
     with_log_at_error(
         fun() -> wolff:stop_and_delete_supervised_producers(Producers) end,
         #{
@@ -103,6 +103,13 @@ on_stop(_InstId, #{client_id := ClientID, producers := Producers}) ->
             msg => "failed_to_delete_kafka_client",
             client_id => ClientID
         }
+    ),
+    with_log_at_error(
+        fun() -> uninstall_telemetry_handlers(InstanceID) end,
+        #{
+            msg => "failed_to_uninstall_telemetry_handlers",
+            client_id => ClientID
+        }
     ).
 
 %% @doc The callback API for rule-engine (or bridge without rules)
@@ -337,12 +344,20 @@ handle_telemetry_event(_EventId, _Metrics, _MetaData, _HandlerConfig) ->
     %% Event that we do not handle
     ok.
 
-maybe_install_wolff_telemetry_handlers() ->
+-spec telemetry_handler_id(emqx_resource:resource_id()) -> binary().
+telemetry_handler_id(InstanceID) ->
+    <<"emqx-bridge-kafka-producer-", InstanceID/binary, "-telemetry-handler">>.
+
+uninstall_telemetry_handlers(InstanceID) ->
+    HandlerID = telemetry_handler_id(InstanceID),
+    telemetry:detach(HandlerID).
+
+maybe_install_wolff_telemetry_handlers(InstanceID) ->
     %% Attach event handlers for Kafka telemetry events. If a handler with the
     %% handler id already exists, the attach_many function does nothing
     telemetry:attach_many(
         %% unique handler id
-        <<"emqx-bridge-kafka-producer-telemetry-handler">>,
+        telemetry_handler_id(InstanceID),
         %% Note: we don't handle `[wolff, success]' because,
         %% currently, we already increment the success counter for
         %% this resource at `emqx_rule_runtime:handle_action' when
@@ -358,6 +373,6 @@ maybe_install_wolff_telemetry_handlers() ->
             [wolff, retried_failed],
             [wolff, retried_success]
         ],
-        fun emqx_bridge_impl_kafka_producer:handle_telemetry_event/4,
+        fun ?MODULE:handle_telemetry_event/4,
         []
     ).

From a6ba8494d8cfd4dd50532b96b29ed36f5741b95a Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 13 Oct 2022 11:16:37 -0300
Subject: [PATCH 182/232] feat: start `telemetry` app before `emqx`

---
 mix.exs          | 1 +
 rebar.config.erl | 1 +
 2 files changed, 2 insertions(+)

diff --git a/mix.exs b/mix.exs
index 5750598ae..e9f861ce5 100644
--- a/mix.exs
+++ b/mix.exs
@@ -208,6 +208,7 @@ defmodule EMQXUmbrella.MixProject do
       redbug: :permanent,
       xmerl: :permanent,
       hocon: :load,
+      telemetry: :permanent,
       emqx: :load,
       emqx_conf: :load,
       emqx_machine: :permanent
diff --git a/rebar.config.erl b/rebar.config.erl
index a8c54e5f4..f745b5cca 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -360,6 +360,7 @@ relx_apps(ReleaseType, Edition) ->
         redbug,
         xmerl,
         {hocon, load},
+        telemetry,
         % started by emqx_machine
         {emqx, load},
         {emqx_conf, load},

From 1b2b629cdde72c433327e2d55f7eef310da42083 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 13 Oct 2022 11:04:47 -0300
Subject: [PATCH 183/232] feat: emit telemetry events for all resource worker
 metrics

---
 apps/emqx_resource/src/emqx_resource.app.src  |   3 +-
 apps/emqx_resource/src/emqx_resource_app.erl  |   9 +
 .../src/emqx_resource_metrics.erl             | 160 +++++++++++++++---
 .../src/emqx_resource_worker.erl              |  13 +-
 .../kafka/emqx_bridge_impl_kafka_producer.erl |   4 +-
 5 files changed, 148 insertions(+), 41 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src
index b688e3c11..38dac5449 100644
--- a/apps/emqx_resource/src/emqx_resource.app.src
+++ b/apps/emqx_resource/src/emqx_resource.app.src
@@ -9,7 +9,8 @@
         stdlib,
         gproc,
         jsx,
-        emqx
+        emqx,
+        telemetry
     ]},
     {env, []},
     {modules, []},
diff --git a/apps/emqx_resource/src/emqx_resource_app.erl b/apps/emqx_resource/src/emqx_resource_app.erl
index 72838a8c1..51e7b2556 100644
--- a/apps/emqx_resource/src/emqx_resource_app.erl
+++ b/apps/emqx_resource/src/emqx_resource_app.erl
@@ -23,9 +23,18 @@
 -export([start/2, stop/1]).
 
 start(_StartType, _StartArgs) ->
+    %% since the handler is generic and executed in the process
+    %% emitting the event, we need to install only a single handler
+    %% for the whole app.
+    TelemetryHandlerID = telemetry_handler_id(),
+    ok = emqx_resource_metrics:install_telemetry_handler(TelemetryHandlerID),
     emqx_resource_sup:start_link().
 
 stop(_State) ->
+    TelemetryHandlerID = telemetry_handler_id(),
+    ok = emqx_resource_metrics:uninstall_telemetry_handler(TelemetryHandlerID),
     ok.
 
 %% internal functions
+telemetry_handler_id() ->
+    <<"emqx-resource-app-telemetry-handler">>.
diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl
index 96d955db0..e6637b68f 100644
--- a/apps/emqx_resource/src/emqx_resource_metrics.erl
+++ b/apps/emqx_resource/src/emqx_resource_metrics.erl
@@ -16,6 +16,13 @@
 
 -module(emqx_resource_metrics).
 
+-export([
+    events/0,
+    install_telemetry_handler/1,
+    uninstall_telemetry_handler/1,
+    handle_telemetry_event/4
+]).
+
 -export([
     batching_change/2,
     batching_get/1,
@@ -62,6 +69,91 @@
 ]).
 
 -define(RES_METRICS, resource_metrics).
+-define(TELEMETRY_PREFIX, emqx, resource).
+
+-spec events() -> [telemetry:event_name()].
+events() ->
+    [
+        [?TELEMETRY_PREFIX, Event]
+     || Event <- [
+            batching,
+            dropped_other,
+            dropped_queue_full,
+            dropped_queue_not_enabled,
+            dropped_resource_not_found,
+            dropped_resource_stopped,
+            failed,
+            inflight,
+            matched,
+            queuing,
+            retried_failed,
+            retried_success,
+            success
+        ]
+    ].
+
+-spec install_telemetry_handler(binary()) -> ok.
+install_telemetry_handler(HandlerID) ->
+    _ = telemetry:attach_many(
+        HandlerID,
+        events(),
+        fun ?MODULE:handle_telemetry_event/4,
+        _HandlerConfig = #{}
+    ),
+    ok.
+
+-spec uninstall_telemetry_handler(binary()) -> ok.
+uninstall_telemetry_handler(HandlerID) ->
+    _ = telemetry:detach(HandlerID),
+    ok.
+
+handle_telemetry_event(
+    [?TELEMETRY_PREFIX, Event],
+    _Measurements = #{counter_inc := Val},
+    _Metadata = #{resource_id := ID},
+    _HandlerConfig
+) ->
+    case Event of
+        batching ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'batching', Val);
+        dropped_other ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.other', Val);
+        dropped_queue_full ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_full', Val);
+        dropped_queue_not_enabled ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_not_enabled', Val);
+        dropped_resource_not_found ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_not_found', Val);
+        dropped_resource_stopped ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_stopped', Val);
+        failed ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed', Val);
+        inflight ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'inflight', Val);
+        matched ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'matched', Val);
+        queuing ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'queuing', Val);
+        retried_failed ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.failed', Val);
+        retried_success ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'success', Val),
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.success', Val);
+        success ->
+            emqx_metrics_worker:inc(?RES_METRICS, ID, 'success', Val);
+        _ ->
+            ok
+    end;
+handle_telemetry_event(_EventName, _Measurements, _Metadata, _HandlerConfig) ->
+    ok.
 
 %% Gauges (value can go both up and down):
 %% --------------------------------------
@@ -69,14 +161,14 @@
 %% @doc Count of messages that are currently accumulated in memory waiting for
 %% being sent in one batch
 batching_change(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'batching', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, batching], #{counter_inc => Val}, #{resource_id => ID}).
 
 batching_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'batching').
 
 %% @doc Count of messages that are currently queuing. [Gauge]
 queuing_change(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'queuing', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, queuing], #{counter_inc => Val}, #{resource_id => ID}).
 
 queuing_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'queuing').
@@ -84,7 +176,7 @@ queuing_get(ID) ->
 %% @doc Count of messages that were sent asynchronously but ACKs are not
 %% received. [Gauge]
 inflight_change(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'inflight', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, inflight], #{counter_inc => Val}, #{resource_id => ID}).
 
 inflight_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'inflight').
@@ -94,120 +186,134 @@ inflight_get(ID) ->
 
 %% @doc Count of messages dropped
 dropped_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped').
+    dropped_inc(ID, 1).
 
 dropped_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, dropped], #{counter_inc => Val}, #{resource_id => ID}).
 
 dropped_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped').
 
 %% @doc Count of messages dropped due to other reasons
 dropped_other_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.other').
+    dropped_other_inc(ID, 1).
 
 dropped_other_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.other', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, dropped_other], #{counter_inc => Val}, #{
+        resource_id => ID
+    }).
 
 dropped_other_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.other').
 
 %% @doc Count of messages dropped because the queue was full
 dropped_queue_full_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_full').
+    dropped_queue_full_inc(ID, 1).
 
 dropped_queue_full_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_full', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, dropped_queue_full], #{counter_inc => Val}, #{
+        resource_id => ID
+    }).
 
 dropped_queue_full_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.queue_full').
 
 %% @doc Count of messages dropped because the queue was not enabled
 dropped_queue_not_enabled_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_not_enabled').
+    dropped_queue_not_enabled_inc(ID, 1).
 
 dropped_queue_not_enabled_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.queue_not_enabled', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, dropped_queue_not_enabled], #{counter_inc => Val}, #{
+        resource_id => ID
+    }).
 
 dropped_queue_not_enabled_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.queue_not_enabled').
 
 %% @doc Count of messages dropped because the resource was not found
 dropped_resource_not_found_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_not_found').
+    dropped_resource_not_found_inc(ID, 1).
 
 dropped_resource_not_found_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_not_found', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, dropped_resource_not_found], #{counter_inc => Val}, #{
+        resource_id => ID
+    }).
 
 dropped_resource_not_found_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.resource_not_found').
 
 %% @doc Count of messages dropped because the resource was stopped
 dropped_resource_stopped_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_stopped').
+    dropped_resource_stopped_inc(ID, 1).
 
 dropped_resource_stopped_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'dropped.resource_stopped', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, dropped_resource_stopped], #{counter_inc => Val}, #{
+        resource_id => ID
+    }).
 
 dropped_resource_stopped_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'dropped.resource_stopped').
 
 %% @doc Count of how many times this bridge has been matched and queried
 matched_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'matched').
+    matched_inc(ID, 1).
 
 matched_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'matched', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, matched], #{counter_inc => Val}, #{resource_id => ID}).
 
 matched_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'matched').
 
 %% @doc The number of times message sends have been retried
 retried_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried').
+    retried_inc(ID, 1).
 
 retried_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, retried], #{counter_inc => Val}, #{resource_id => ID}).
 
 retried_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'retried').
 
 %% @doc Count of message sends that have failed
 failed_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed').
+    failed_inc(ID, 1).
 
 failed_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'failed', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, failed], #{counter_inc => Val}, #{resource_id => ID}).
 
 failed_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'failed').
 
 %%% @doc Count of message sends that have failed after having been retried
 retried_failed_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.failed').
+    retried_failed_inc(ID, 1).
 
 retried_failed_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.failed', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, retried_failed], #{counter_inc => Val}, #{
+        resource_id => ID
+    }).
 
 retried_failed_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'retried.failed').
 
 %% @doc Count messages that were sucessfully sent after at least one retry
 retried_success_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.success').
+    retried_success_inc(ID, 1).
 
 retried_success_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'retried.success', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, retried_success], #{counter_inc => Val}, #{
+        resource_id => ID
+    }).
 
 retried_success_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'retried.success').
 
 %% @doc Count of messages that have been sent successfully
 success_inc(ID) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'success').
+    success_inc(ID, 1).
 
 success_inc(ID, Val) ->
-    emqx_metrics_worker:inc(?RES_METRICS, ID, 'success', Val).
+    telemetry:execute([?TELEMETRY_PREFIX, success], #{counter_inc => Val}, #{resource_id => ID}).
 
 success_get(ID) ->
     emqx_metrics_worker:get(?RES_METRICS, ID, 'success').
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index 75a63d427..b309299f6 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -376,17 +376,14 @@ handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasSent, _) when
     true;
 handle_query_result(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => resource_not_found, info => Msg}),
-    emqx_resource_metrics:dropped_inc(Id),
     emqx_resource_metrics:dropped_resource_not_found_inc(Id),
     BlockWorker;
 handle_query_result(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => resource_stopped, info => Msg}),
-    emqx_resource_metrics:dropped_inc(Id),
     emqx_resource_metrics:dropped_resource_stopped_inc(Id),
     BlockWorker;
 handle_query_result(Id, ?RESOURCE_ERROR_M(Reason, _), _HasSent, BlockWorker) ->
     ?SLOG(error, #{id => Id, msg => other_resource_error, reason => Reason}),
-    emqx_resource_metrics:dropped_inc(Id),
     emqx_resource_metrics:dropped_other_inc(Id),
     BlockWorker;
 handle_query_result(Id, {error, {recoverable_error, Reason}}, _HasSent, _BlockWorker) ->
@@ -546,7 +543,6 @@ estimate_size(QItem) ->
     size(queue_item_marshaller(QItem)).
 
 maybe_append_queue(Id, undefined, _Items) ->
-    emqx_resource_metrics:dropped_inc(Id),
     emqx_resource_metrics:dropped_queue_not_enabled_inc(Id),
     undefined;
 maybe_append_queue(Id, Q, Items) ->
@@ -560,7 +556,6 @@ maybe_append_queue(Id, Q, Items) ->
                 ok = replayq:ack(Q1, QAckRef),
                 Dropped = length(Items2),
                 emqx_resource_metrics:queuing_change(Id, -Dropped),
-                emqx_resource_metrics:dropped_inc(Id),
                 emqx_resource_metrics:dropped_queue_full_inc(Id),
                 ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
                 Q1
@@ -641,16 +636,12 @@ inflight_drop(Name, Ref) ->
 
 %%==============================================================================
 
-inc_sent_failed(Id, true) ->
-    emqx_resource_metrics:failed_inc(Id),
-    emqx_resource_metrics:retried_inc(Id),
+inc_sent_failed(Id, _HasSent = true) ->
     emqx_resource_metrics:retried_failed_inc(Id);
 inc_sent_failed(Id, _HasSent) ->
     emqx_resource_metrics:failed_inc(Id).
 
-inc_sent_success(Id, true) ->
-    emqx_resource_metrics:success_inc(Id),
-    emqx_resource_metrics:retried_inc(Id),
+inc_sent_success(Id, _HasSent = true) ->
     emqx_resource_metrics:retried_success_inc(Id);
 inc_sent_success(Id, _HasSent) ->
     emqx_resource_metrics:success_inc(Id).
diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
index 0a03f582f..19f110bd9 100644
--- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -90,14 +90,14 @@ on_start(InstId, Config) ->
     end.
 
 on_stop(InstanceID, #{client_id := ClientID, producers := Producers}) ->
-    with_log_at_error(
+    _ = with_log_at_error(
         fun() -> wolff:stop_and_delete_supervised_producers(Producers) end,
         #{
             msg => "failed_to_delete_kafka_producer",
             client_id => ClientID
         }
     ),
-    with_log_at_error(
+    _ = with_log_at_error(
         fun() -> wolff:stop_and_delete_supervised_client(ClientID) end,
         #{
             msg => "failed_to_delete_kafka_client",

From 2d01726b223dbcf8c33172bed3ae9fd873d77a10 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 13 Oct 2022 15:27:35 -0300
Subject: [PATCH 184/232] fix: account calls when resource is not connected as
 matched

---
 .../test/emqx_bridge_mqtt_SUITE.erl           | 46 ++++++++++++++-----
 .../src/emqx_resource_worker.erl              |  3 ++
 2 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
index 819556d81..84152efc6 100644
--- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -23,6 +23,7 @@
 -include("emqx/include/emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include("emqx_dashboard/include/emqx_dashboard.hrl").
 
 %% output functions
@@ -511,15 +512,15 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
     %% we now test if the bridge works as expected
     LocalTopic = <<"local_topic/1">>,
     RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
-    Payload = <<"hello">>,
+    Payload0 = <<"hello">>,
     emqx:subscribe(RemoteTopic),
     timer:sleep(100),
     %% PUBLISH a message to the 'local' broker, as we have only one broker,
     %% the remote broker is also the local one.
-    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+    emqx:publish(emqx_message:make(LocalTopic, Payload0)),
 
     %% we should receive a message on the "remote" broker, with specified topic
-    assert_mqtt_msg_received(RemoteTopic, Payload),
+    assert_mqtt_msg_received(RemoteTopic, Payload0),
 
     %% verify the metrics of the bridge
     {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
@@ -543,18 +544,40 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
     ct:sleep(1500),
 
     %% PUBLISH 2 messages to the 'local' broker, the message should
-    emqx:publish(emqx_message:make(LocalTopic, Payload)),
-    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+    ok = snabbkaffe:start_trace(),
+    {ok, SRef} =
+        snabbkaffe:subscribe(
+            fun
+                (
+                    #{
+                        ?snk_kind := call_query_enter,
+                        query := {query, _From, {send_message, #{}}, _Sent}
+                    }
+                ) ->
+                    true;
+                (_) ->
+                    false
+            end,
+            _NEvents = 2,
+            _Timeout = 1_000
+        ),
+    Payload1 = <<"hello2">>,
+    Payload2 = <<"hello3">>,
+    emqx:publish(emqx_message:make(LocalTopic, Payload1)),
+    emqx:publish(emqx_message:make(LocalTopic, Payload2)),
+    {ok, _} = snabbkaffe:receive_events(SRef),
+    ok = snabbkaffe:stop(),
 
     %% verify the metrics of the bridge, the message should be queued
     {ok, 200, BridgeStr1} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    %% matched >= 3 because of possible retries.
     ?assertMatch(
         #{
             <<"status">> := Status,
             <<"metrics">> := #{
-                <<"matched">> := 1, <<"success">> := 1, <<"failed">> := 0, <<"queuing">> := 2
+                <<"matched">> := Matched, <<"success">> := 1, <<"failed">> := 0, <<"queuing">> := 2
             }
-        } when Status == <<"connected">> orelse Status == <<"connecting">>,
+        } when Matched >= 3 andalso (Status == <<"connected">> orelse Status == <<"connecting">>),
         jsx:decode(BridgeStr1)
     ),
 
@@ -563,22 +586,23 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
     timer:sleep(1500),
     %% verify the metrics of the bridge, the 2 queued messages should have been sent
     {ok, 200, BridgeStr2} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    %% matched >= 3 because of possible retries.
     ?assertMatch(
         #{
             <<"status">> := <<"connected">>,
             <<"metrics">> := #{
-                <<"matched">> := 3,
+                <<"matched">> := Matched,
                 <<"success">> := 3,
                 <<"failed">> := 0,
                 <<"queuing">> := 0,
                 <<"retried">> := _
             }
-        },
+        } when Matched >= 3,
         jsx:decode(BridgeStr2)
     ),
     %% also verify the 2 messages have been sent to the remote broker
-    assert_mqtt_msg_received(RemoteTopic, Payload),
-    assert_mqtt_msg_received(RemoteTopic, Payload),
+    assert_mqtt_msg_received(RemoteTopic, Payload1),
+    assert_mqtt_msg_received(RemoteTopic, Payload2),
     %% delete the bridge
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl
index b309299f6..a36cb15b7 100644
--- a/apps/emqx_resource/src/emqx_resource_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_worker.erl
@@ -410,6 +410,7 @@ handle_query_result(Id, Result, HasSent, BlockWorker) ->
     BlockWorker.
 
 call_query(QM0, Id, Query, QueryOpts) ->
+    ?tp(call_query_enter, #{id => Id, query => Query}),
     case emqx_resource_manager:ets_lookup(Id) of
         {ok, _Group, #{mod := Mod, state := ResSt, status := connected} = Data} ->
             QM =
@@ -421,8 +422,10 @@ call_query(QM0, Id, Query, QueryOpts) ->
             emqx_resource_metrics:matched_inc(Id),
             apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt, QueryOpts);
         {ok, _Group, #{status := stopped}} ->
+            emqx_resource_metrics:matched_inc(Id),
             ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
         {ok, _Group, #{status := S}} when S == connecting; S == disconnected ->
+            emqx_resource_metrics:matched_inc(Id),
             ?RESOURCE_ERROR(not_connected, "resource not connected");
         {error, not_found} ->
             ?RESOURCE_ERROR(not_found, "resource not found")

From ee4c723fcb08420c1fd8d80d9de0935fca92a5d7 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 17 Oct 2022 16:26:04 -0300
Subject: [PATCH 185/232] refactor: simplify wolff telemetry handler id

---
 .../src/kafka/emqx_bridge_impl_kafka_producer.erl               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
index 19f110bd9..aceafd6d1 100644
--- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -346,7 +346,7 @@ handle_telemetry_event(_EventId, _Metrics, _MetaData, _HandlerConfig) ->
 
 -spec telemetry_handler_id(emqx_resource:resource_id()) -> binary().
 telemetry_handler_id(InstanceID) ->
-    <<"emqx-bridge-kafka-producer-", InstanceID/binary, "-telemetry-handler">>.
+    <<"emqx-bridge-kafka-producer-", InstanceID/binary>>.
 
 uninstall_telemetry_handlers(InstanceID) ->
     HandlerID = telemetry_handler_id(InstanceID),

From 62eeb4b8e8290f5e9869fe987fdb9c2d872f26ee Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 18 Oct 2022 09:32:35 -0300
Subject: [PATCH 186/232] feat(resource): reset metrics when stopping a
 resource

---
 .../emqx_resource/src/emqx_resource_manager.erl |  5 ++++-
 apps/emqx_resource/test/emqx_resource_SUITE.erl | 17 ++++++++++++++++-
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index c8d5e4194..10c501865 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -507,13 +507,16 @@ start_resource(Data, From) ->
 
 stop_resource(#data{state = undefined, id = ResId} = _Data) ->
     _ = maybe_clear_alarm(ResId),
+    ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId),
     ok;
 stop_resource(Data) ->
     %% We don't care the return value of the Mod:on_stop/2.
     %% The callback mod should make sure the resource is stopped after on_stop/2
     %% is returned.
+    ResId = Data#data.id,
     _ = emqx_resource:call_stop(Data#data.manager_id, Data#data.mod, Data#data.state),
-    _ = maybe_clear_alarm(Data#data.id),
+    _ = maybe_clear_alarm(ResId),
+    ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId),
     ok.
 
 make_test_id() ->
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 672e01896..107ca2a93 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -502,6 +502,10 @@ t_stop_start(_) ->
         #{<<"name">> => <<"test_resource">>}
     ),
 
+    %% add some metrics to test their persistence
+    emqx_resource_metrics:batching_change(?ID, 5),
+    ?assertEqual(5, emqx_resource_metrics:batching_get(?ID)),
+
     {ok, _} = emqx_resource:check_and_recreate(
         ?ID,
         ?TEST_RESOURCE,
@@ -513,6 +517,9 @@ t_stop_start(_) ->
 
     ?assert(is_process_alive(Pid0)),
 
+    %% metrics are reset when recreating
+    ?assertEqual(0, emqx_resource_metrics:batching_get(?ID)),
+
     ok = emqx_resource:stop(?ID),
 
     ?assertNot(is_process_alive(Pid0)),
@@ -527,7 +534,15 @@ t_stop_start(_) ->
 
     {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state),
 
-    ?assert(is_process_alive(Pid1)).
+    ?assert(is_process_alive(Pid1)),
+
+    %% now stop while resetting the metrics
+    emqx_resource_metrics:batching_change(?ID, 5),
+    ?assertEqual(5, emqx_resource_metrics:batching_get(?ID)),
+    ok = emqx_resource:stop(?ID),
+    ?assertEqual(0, emqx_resource_metrics:batching_get(?ID)),
+
+    ok.
 
 t_stop_start_local(_) ->
     {error, _} = emqx_resource:check_and_create_local(

From b6445cdc14583701bd61c934dd318dda67677b81 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 1 Nov 2022 16:53:07 -0300
Subject: [PATCH 187/232] test: remove unused var warning

---
 apps/emqx/test/emqx_shared_sub_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl
index f53e8f374..8616028ca 100644
--- a/apps/emqx/test/emqx_shared_sub_SUITE.erl
+++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl
@@ -594,7 +594,7 @@ t_remote(_) ->
 
     try
         {ok, ClientPidLocal} = emqtt:connect(ConnPidLocal),
-        {ok, ClientPidRemote} = emqtt:connect(ConnPidRemote),
+        {ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote),
 
         emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}),
 

From 04588148b7ad20f51f69076a2e5a3bf78583746e Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 1 Nov 2022 16:39:11 -0300
Subject: [PATCH 188/232] test(influxdb): increase influxdb bridge/connector
 coverage (ee5.0)

---
 .../docker-compose-influxdb-tcp.yaml          |  36 +
 .../docker-compose-influxdb-tls.yaml          |  42 +
 .../docker-compose-toxiproxy.yaml             |  18 +
 .ci/docker-compose-file/influxdb/setup-v1.sh  |  16 +
 .ci/docker-compose-file/toxiproxy.json        |  14 +
 .github/workflows/run_test_cases.yaml         |   1 +
 Makefile                                      |   4 +
 apps/emqx/test/emqx_common_test_helpers.erl   | 135 +++
 lib-ee/emqx_ee_bridge/docker-ct               |   3 +-
 .../test/emqx_ee_bridge_influxdb_SUITE.erl    | 863 ++++++++++++++++++
 .../src/emqx_ee_connector_influxdb.erl        | 117 ++-
 rebar.config.erl                              |   3 +-
 scripts/ct/run.sh                             |   5 +
 13 files changed, 1249 insertions(+), 8 deletions(-)
 create mode 100644 .ci/docker-compose-file/docker-compose-influxdb-tcp.yaml
 create mode 100644 .ci/docker-compose-file/docker-compose-influxdb-tls.yaml
 create mode 100644 .ci/docker-compose-file/docker-compose-toxiproxy.yaml
 create mode 100755 .ci/docker-compose-file/influxdb/setup-v1.sh
 create mode 100644 .ci/docker-compose-file/toxiproxy.json
 create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl

diff --git a/.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml b/.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml
new file mode 100644
index 000000000..1780bc7e2
--- /dev/null
+++ b/.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml
@@ -0,0 +1,36 @@
+version: '3.9'
+
+services:
+  influxdb_server_tcp:
+    container_name: influxdb_tcp
+    image: influxdb:${INFLUXDB_TAG}
+    expose:
+      - "8086"
+      - "8089/udp"
+      - "8083"
+    # ports:
+    #   - "8086:8086"
+    environment:
+      DOCKER_INFLUXDB_INIT_MODE: setup
+      DOCKER_INFLUXDB_INIT_USERNAME: root
+      DOCKER_INFLUXDB_INIT_PASSWORD: emqx@123
+      DOCKER_INFLUXDB_INIT_ORG: emqx
+      DOCKER_INFLUXDB_INIT_BUCKET: mqtt
+      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: abcdefg
+    volumes:
+      - "./influxdb/setup-v1.sh:/docker-entrypoint-initdb.d/setup-v1.sh"
+    restart: always
+    networks:
+      - emqx_bridge
+
+# networks:
+#   emqx_bridge:
+#     driver: bridge
+#     name: emqx_bridge
+#     ipam:
+#       driver: default
+#       config:
+#         - subnet: 172.100.239.0/24
+#           gateway: 172.100.239.1
+#         - subnet: 2001:3200:3200::/64
+#           gateway: 2001:3200:3200::1
diff --git a/.ci/docker-compose-file/docker-compose-influxdb-tls.yaml b/.ci/docker-compose-file/docker-compose-influxdb-tls.yaml
new file mode 100644
index 000000000..ec1600bf2
--- /dev/null
+++ b/.ci/docker-compose-file/docker-compose-influxdb-tls.yaml
@@ -0,0 +1,42 @@
+version: '3.9'
+
+services:
+  influxdb_server_tls:
+    container_name: influxdb_tls
+    image: influxdb:${INFLUXDB_TAG}
+    expose:
+      - "8086"
+      - "8089/udp"
+      - "8083"
+    # ports:
+    #   - "8087:8086"
+    environment:
+      DOCKER_INFLUXDB_INIT_MODE: setup
+      DOCKER_INFLUXDB_INIT_USERNAME: root
+      DOCKER_INFLUXDB_INIT_PASSWORD: emqx@123
+      DOCKER_INFLUXDB_INIT_ORG: emqx
+      DOCKER_INFLUXDB_INIT_BUCKET: mqtt
+      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: abcdefg
+    volumes:
+      - ./certs/server.crt:/etc/influxdb/cert.pem
+      - ./certs/server.key:/etc/influxdb/key.pem
+      - "./influxdb/setup-v1.sh:/docker-entrypoint-initdb.d/setup-v1.sh"
+    command:
+      - influxd
+      - --tls-cert=/etc/influxdb/cert.pem
+      - --tls-key=/etc/influxdb/key.pem
+    restart: always
+    networks:
+      - emqx_bridge
+
+# networks:
+#   emqx_bridge:
+#     driver: bridge
+#     name: emqx_bridge
+#     ipam:
+#       driver: default
+#       config:
+#         - subnet: 172.100.239.0/24
+#           gateway: 172.100.239.1
+#         - subnet: 2001:3200:3200::/64
+#           gateway: 2001:3200:3200::1
diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
new file mode 100644
index 000000000..924c9e6ae
--- /dev/null
+++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
@@ -0,0 +1,18 @@
+version: '3.9'
+
+services:
+  toxiproxy:
+    container_name: toxiproxy
+    image: ghcr.io/shopify/toxiproxy:2.5.0
+    restart: always
+    networks:
+      - emqx_bridge
+    volumes:
+      - "./toxiproxy.json:/config/toxiproxy.json"
+    ports:
+      - 8474:8474
+      - 8086:8086
+      - 8087:8087
+    command:
+      - "-host=0.0.0.0"
+      - "-config=/config/toxiproxy.json"
diff --git a/.ci/docker-compose-file/influxdb/setup-v1.sh b/.ci/docker-compose-file/influxdb/setup-v1.sh
new file mode 100755
index 000000000..92baf9905
--- /dev/null
+++ b/.ci/docker-compose-file/influxdb/setup-v1.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+# influx v1 dbrp create \
+#   --bucket-id ${DOCKER_INFLUXDB_INIT_BUCKET_ID} \
+#   --db ${V1_DB_NAME} \
+#   --rp ${V1_RP_NAME} \
+#   --default \
+#   --org ${DOCKER_INFLUXDB_INIT_ORG}
+
+influx v1 auth create \
+  --username "${DOCKER_INFLUXDB_INIT_USERNAME}" \
+  --password "${DOCKER_INFLUXDB_INIT_PASSWORD}" \
+  --write-bucket "${DOCKER_INFLUXDB_INIT_BUCKET_ID}" \
+  --org "${DOCKER_INFLUXDB_INIT_ORG}"
diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json
new file mode 100644
index 000000000..176b35d36
--- /dev/null
+++ b/.ci/docker-compose-file/toxiproxy.json
@@ -0,0 +1,14 @@
+[
+  {
+    "name": "influxdb_tcp",
+    "listen": "0.0.0.0:8086",
+    "upstream": "influxdb_tcp:8086",
+    "enabled": true
+  },
+  {
+    "name": "influxdb_tls",
+    "listen": "0.0.0.0:8087",
+    "upstream": "influxdb_tls:8086",
+    "enabled": true
+  }
+]
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 540383ed6..89480f3e5 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -120,6 +120,7 @@ jobs:
             MYSQL_TAG: 8
             PGSQL_TAG: 13
             REDIS_TAG: 6
+            INFLUXDB_TAG: 2.5.0
           run: |
             ./scripts/ct/run.sh --app ${{ matrix.app_name }}
         - uses: actions/upload-artifact@v1
diff --git a/Makefile b/Makefile
index 3dd11aec3..d06becc5d 100644
--- a/Makefile
+++ b/Makefile
@@ -83,7 +83,11 @@ $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app))))
 .PHONY: ct-suite
 ct-suite: $(REBAR)
 ifneq ($(TESTCASE),)
+ifneq ($(GROUP),)
+	$(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE)  --case $(TESTCASE) --group $(GROUP)
+else
 	$(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE)  --case $(TESTCASE)
+endif
 else ifneq ($(GROUP),)
 	$(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE)  --group $(GROUP)
 else
diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 8c0ed2377..196546b7b 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -65,6 +65,15 @@
     stop_slave/1
 ]).
 
+-export([clear_screen/0]).
+-export([with_mock/4]).
+
+%% Toxiproxy API
+-export([
+    with_failure/5,
+    reset_proxy/2
+]).
+
 -define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])).
 
 -define(MQTT_SSL_TWOWAY, [
@@ -769,3 +778,129 @@ expand_node_specs(Specs, CommonOpts) ->
         end,
         Specs
     ).
+
+%% is useful when iterating on the tests in a loop, to get rid of all
+%% the garbaged printed before the test itself beings.
+clear_screen() ->
+    io:format(standard_io, "\033[H\033[2J", []),
+    io:format(standard_error, "\033[H\033[2J", []),
+    io:format(standard_io, "\033[H\033[3J", []),
+    io:format(standard_error, "\033[H\033[3J", []),
+    ok.
+
+with_mock(Mod, FnName, MockedFn, Fun) ->
+    ok = meck:new(Mod, [non_strict, no_link, no_history, passthrough]),
+    ok = meck:expect(Mod, FnName, MockedFn),
+    try
+        Fun()
+    after
+        ok = meck:unload(Mod)
+    end.
+
+%%-------------------------------------------------------------------------------
+%% Toxiproxy utils
+%%-------------------------------------------------------------------------------
+
+reset_proxy(ProxyHost, ProxyPort) ->
+    Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/reset",
+    Body = <<>>,
+    {ok, {{_, 204, _}, _, _}} = httpc:request(
+        post,
+        {Url, [], "application/json", Body},
+        [],
+        [{body_format, binary}]
+    ).
+
+with_failure(FailureType, Name, ProxyHost, ProxyPort, Fun) ->
+    enable_failure(FailureType, Name, ProxyHost, ProxyPort),
+    try
+        Fun()
+    after
+        heal_failure(FailureType, Name, ProxyHost, ProxyPort)
+    end.
+
+enable_failure(FailureType, Name, ProxyHost, ProxyPort) ->
+    case FailureType of
+        down -> switch_proxy(off, Name, ProxyHost, ProxyPort);
+        timeout -> timeout_proxy(on, Name, ProxyHost, ProxyPort);
+        latency_up -> latency_up_proxy(on, Name, ProxyHost, ProxyPort)
+    end.
+
+heal_failure(FailureType, Name, ProxyHost, ProxyPort) ->
+    case FailureType of
+        down -> switch_proxy(on, Name, ProxyHost, ProxyPort);
+        timeout -> timeout_proxy(off, Name, ProxyHost, ProxyPort);
+        latency_up -> latency_up_proxy(off, Name, ProxyHost, ProxyPort)
+    end.
+
+switch_proxy(Switch, Name, ProxyHost, ProxyPort) ->
+    Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name,
+    Body =
+        case Switch of
+            off -> <<"{\"enabled\":false}">>;
+            on -> <<"{\"enabled\":true}">>
+        end,
+    {ok, {{_, 200, _}, _, _}} = httpc:request(
+        post,
+        {Url, [], "application/json", Body},
+        [],
+        [{body_format, binary}]
+    ).
+
+timeout_proxy(on, Name, ProxyHost, ProxyPort) ->
+    Url =
+        "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name ++
+            "/toxics",
+    NameBin = list_to_binary(Name),
+    Body =
+        <<"{\"name\":\"", NameBin/binary,
+            "_timeout\",\"type\":\"timeout\","
+            "\"stream\":\"upstream\",\"toxicity\":1.0,"
+            "\"attributes\":{\"timeout\":0}}">>,
+    {ok, {{_, 200, _}, _, _}} = httpc:request(
+        post,
+        {Url, [], "application/json", Body},
+        [],
+        [{body_format, binary}]
+    );
+timeout_proxy(off, Name, ProxyHost, ProxyPort) ->
+    ToxicName = Name ++ "_timeout",
+    Url =
+        "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name ++
+            "/toxics/" ++ ToxicName,
+    Body = <<>>,
+    {ok, {{_, 204, _}, _, _}} = httpc:request(
+        delete,
+        {Url, [], "application/json", Body},
+        [],
+        [{body_format, binary}]
+    ).
+
+latency_up_proxy(on, Name, ProxyHost, ProxyPort) ->
+    Url =
+        "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name ++
+            "/toxics",
+    NameBin = list_to_binary(Name),
+    Body =
+        <<"{\"name\":\"", NameBin/binary,
+            "_latency_up\",\"type\":\"latency\","
+            "\"stream\":\"upstream\",\"toxicity\":1.0,"
+            "\"attributes\":{\"latency\":20000,\"jitter\":3000}}">>,
+    {ok, {{_, 200, _}, _, _}} = httpc:request(
+        post,
+        {Url, [], "application/json", Body},
+        [],
+        [{body_format, binary}]
+    );
+latency_up_proxy(off, Name, ProxyHost, ProxyPort) ->
+    ToxicName = Name ++ "_latency_up",
+    Url =
+        "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name ++
+            "/toxics/" ++ ToxicName,
+    Body = <<>>,
+    {ok, {{_, 204, _}, _, _}} = httpc:request(
+        delete,
+        {Url, [], "application/json", Body},
+        [],
+        [{body_format, binary}]
+    ).
diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct
index a79037903..3be129d94 100644
--- a/lib-ee/emqx_ee_bridge/docker-ct
+++ b/lib-ee/emqx_ee_bridge/docker-ct
@@ -1,3 +1,4 @@
+influxdb
+kafka
 mongo
 mongo_rs_sharded
-kafka
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
new file mode 100644
index 000000000..a05a26e29
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
@@ -0,0 +1,863 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_influxdb_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+%%------------------------------------------------------------------------------
+%% CT boilerplate
+%%------------------------------------------------------------------------------
+
+all() ->
+    [
+        {group, with_batch},
+        {group, without_batch}
+    ].
+
+groups() ->
+    TCs = emqx_common_test_helpers:all(?MODULE),
+    [
+        {with_batch, [
+            {group, sync_query},
+            {group, async_query}
+        ]},
+        {without_batch, [
+            {group, sync_query},
+            {group, async_query}
+        ]},
+        {sync_query, [
+            {group, apiv1_tcp},
+            {group, apiv1_tls},
+            {group, apiv2_tcp},
+            {group, apiv2_tls}
+        ]},
+        {async_query, [
+            {group, apiv1_tcp},
+            {group, apiv1_tls},
+            {group, apiv2_tcp},
+            {group, apiv2_tls}
+        ]},
+        {apiv1_tcp, TCs},
+        {apiv1_tls, TCs},
+        {apiv2_tcp, TCs},
+        {apiv2_tls, TCs}
+    ].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
+    ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource]),
+    _ = application:stop(emqx_connector),
+    ok.
+
+init_per_group(InfluxDBType, Config0) when
+    InfluxDBType =:= apiv1_tcp;
+    InfluxDBType =:= apiv1_tls
+->
+    #{
+        host := InfluxDBHost,
+        port := InfluxDBPort,
+        use_tls := UseTLS,
+        proxy_name := ProxyName
+    } =
+        case InfluxDBType of
+            apiv1_tcp ->
+                #{
+                    host => os:getenv("INFLUXDB_APIV1_TCP_HOST", "toxiproxy"),
+                    port => list_to_integer(os:getenv("INFLUXDB_APIV1_TCP_PORT", "8086")),
+                    use_tls => false,
+                    proxy_name => "influxdb_tcp"
+                };
+            apiv1_tls ->
+                #{
+                    host => os:getenv("INFLUXDB_APIV1_TLS_HOST", "toxiproxy"),
+                    port => list_to_integer(os:getenv("INFLUXDB_APIV1_TLS_PORT", "8087")),
+                    use_tls => true,
+                    proxy_name => "influxdb_tls"
+                }
+        end,
+    case emqx_common_test_helpers:is_tcp_server_available(InfluxDBHost, InfluxDBPort) of
+        true ->
+            ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
+            ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
+            emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge]),
+            {ok, _} = application:ensure_all_started(emqx_connector),
+            Config = [{use_tls, UseTLS} | Config0],
+            {Name, ConfigString, InfluxDBConfig} = influxdb_config(
+                apiv1, InfluxDBHost, InfluxDBPort, Config
+            ),
+            EHttpcPoolNameBin = <<(atom_to_binary(?MODULE))/binary, "_apiv1">>,
+            EHttpcPoolName = binary_to_atom(EHttpcPoolNameBin),
+            {EHttpcTransport, EHttpcTransportOpts} =
+                case UseTLS of
+                    true -> {tls, [{verify, verify_none}]};
+                    false -> {tcp, []}
+                end,
+            EHttpcPoolOpts = [
+                {host, InfluxDBHost},
+                {port, InfluxDBPort},
+                {pool_size, 1},
+                {transport, EHttpcTransport},
+                {transport_opts, EHttpcTransportOpts}
+            ],
+            {ok, _} = ehttpc_sup:start_pool(EHttpcPoolName, EHttpcPoolOpts),
+            [
+                {proxy_host, ProxyHost},
+                {proxy_port, ProxyPort},
+                {proxy_name, ProxyName},
+                {influxdb_host, InfluxDBHost},
+                {influxdb_port, InfluxDBPort},
+                {influxdb_type, apiv1},
+                {influxdb_config, InfluxDBConfig},
+                {influxdb_config_string, ConfigString},
+                {ehttpc_pool_name, EHttpcPoolName},
+                {influxdb_name, Name}
+                | Config
+            ];
+        false ->
+            {skip, no_influxdb}
+    end;
+init_per_group(InfluxDBType, Config0) when
+    InfluxDBType =:= apiv2_tcp;
+    InfluxDBType =:= apiv2_tls
+->
+    #{
+        host := InfluxDBHost,
+        port := InfluxDBPort,
+        use_tls := UseTLS,
+        proxy_name := ProxyName
+    } =
+        case InfluxDBType of
+            apiv2_tcp ->
+                #{
+                    host => os:getenv("INFLUXDB_APIV2_TCP_HOST", "toxiproxy"),
+                    port => list_to_integer(os:getenv("INFLUXDB_APIV2_TCP_PORT", "8086")),
+                    use_tls => false,
+                    proxy_name => "influxdb_tcp"
+                };
+            apiv2_tls ->
+                #{
+                    host => os:getenv("INFLUXDB_APIV2_TLS_HOST", "toxiproxy"),
+                    port => list_to_integer(os:getenv("INFLUXDB_APIV2_TLS_PORT", "8087")),
+                    use_tls => true,
+                    proxy_name => "influxdb_tls"
+                }
+        end,
+    case emqx_common_test_helpers:is_tcp_server_available(InfluxDBHost, InfluxDBPort) of
+        true ->
+            ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
+            ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
+            emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge]),
+            {ok, _} = application:ensure_all_started(emqx_connector),
+            Config = [{use_tls, UseTLS} | Config0],
+            {Name, ConfigString, InfluxDBConfig} = influxdb_config(
+                apiv2, InfluxDBHost, InfluxDBPort, Config
+            ),
+            EHttpcPoolNameBin = <<(atom_to_binary(?MODULE))/binary, "_apiv2">>,
+            EHttpcPoolName = binary_to_atom(EHttpcPoolNameBin),
+            {EHttpcTransport, EHttpcTransportOpts} =
+                case UseTLS of
+                    true -> {tls, [{verify, verify_none}]};
+                    false -> {tcp, []}
+                end,
+            EHttpcPoolOpts = [
+                {host, InfluxDBHost},
+                {port, InfluxDBPort},
+                {pool_size, 1},
+                {transport, EHttpcTransport},
+                {transport_opts, EHttpcTransportOpts}
+            ],
+            {ok, _} = ehttpc_sup:start_pool(EHttpcPoolName, EHttpcPoolOpts),
+            [
+                {proxy_host, ProxyHost},
+                {proxy_port, ProxyPort},
+                {proxy_name, ProxyName},
+                {influxdb_host, InfluxDBHost},
+                {influxdb_port, InfluxDBPort},
+                {influxdb_type, apiv2},
+                {influxdb_config, InfluxDBConfig},
+                {influxdb_config_string, ConfigString},
+                {ehttpc_pool_name, EHttpcPoolName},
+                {influxdb_name, Name}
+                | Config
+            ];
+        false ->
+            {skip, no_influxdb}
+    end;
+init_per_group(sync_query, Config) ->
+    [{query_mode, sync} | Config];
+init_per_group(async_query, Config) ->
+    [{query_mode, async} | Config];
+init_per_group(with_batch, Config) ->
+    [{enable_batch, true} | Config];
+init_per_group(without_batch, Config) ->
+    [{enable_batch, false} | Config];
+init_per_group(_Group, Config) ->
+    Config.
+
+end_per_group(Group, Config) when
+    Group =:= apiv1_tcp;
+    Group =:= apiv1_tls;
+    Group =:= apiv2_tcp;
+    Group =:= apiv2_tls
+->
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    EHttpcPoolName = ?config(ehttpc_pool_name, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    ehttpc_sup:stop_pool(EHttpcPoolName),
+    delete_bridge(Config),
+    ok;
+end_per_group(_Group, _Config) ->
+    ok.
+
+init_per_testcase(_Testcase, Config) ->
+    %% catch clear_db(Config),
+    %% delete_bridge(Config),
+    delete_all_bridges(),
+    Config.
+
+end_per_testcase(_Testcase, Config) ->
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    %% catch clear_db(Config),
+    %% delete_bridge(Config),
+    delete_all_bridges(),
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Helper fns
+%%------------------------------------------------------------------------------
+
+example_write_syntax() ->
+    %% N.B.: this single space character is relevant
+    <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
+        "${clientid}_int_value=${payload.int_key}i,",
+        "uint_value=${payload.uint_key}u,"
+        "float_value=${payload.float_key},", "undef_value=${payload.undef},",
+        "${undef_key}=\"hard-coded-value\",", "bool=${payload.bool}">>.
+
+influxdb_config(apiv1 = Type, InfluxDBHost, InfluxDBPort, Config) ->
+    EnableBatch = proplists:get_value(enable_batch, Config, true),
+    QueryMode = proplists:get_value(query_mode, Config, sync),
+    UseTLS = proplists:get_value(use_tls, Config, false),
+    Name = atom_to_binary(?MODULE),
+    WriteSyntax = example_write_syntax(),
+    ConfigString =
+        io_lib:format(
+            "bridges.influxdb_api_v1.~s {\n"
+            "  enable = true\n"
+            "  server = \"~p:~b\"\n"
+            "  database = mqtt\n"
+            "  username = root\n"
+            "  password = emqx@123\n"
+            "  precision = ns\n"
+            "  write_syntax = \"~s\"\n"
+            "  resource_opts = {\n"
+            "    enable_batch = ~p\n"
+            "    query_mode = ~s\n"
+            "  }\n"
+            "  ssl {\n"
+            "    enable = ~p\n"
+            "    verify = verify_none\n"
+            "  }\n"
+            "}\n",
+            [Name, InfluxDBHost, InfluxDBPort, WriteSyntax, EnableBatch, QueryMode, UseTLS]
+        ),
+    {Name, ConfigString, parse_and_check(ConfigString, Type, Name)};
+influxdb_config(apiv2 = Type, InfluxDBHost, InfluxDBPort, Config) ->
+    EnableBatch = proplists:get_value(enable_batch, Config, true),
+    QueryMode = proplists:get_value(query_mode, Config, sync),
+    UseTLS = proplists:get_value(use_tls, Config, false),
+    Name = atom_to_binary(?MODULE),
+    WriteSyntax = example_write_syntax(),
+    ConfigString =
+        io_lib:format(
+            "bridges.influxdb_api_v2.~s {\n"
+            "  enable = true\n"
+            "  server = \"~p:~b\"\n"
+            "  bucket = mqtt\n"
+            "  org = emqx\n"
+            "  token = abcdefg\n"
+            "  precision = ns\n"
+            "  write_syntax = \"~s\"\n"
+            "  resource_opts = {\n"
+            "    enable_batch = ~p\n"
+            "    query_mode = ~s\n"
+            "  }\n"
+            "  ssl {\n"
+            "    enable = ~p\n"
+            "    verify = verify_none\n"
+            "  }\n"
+            "}\n",
+            [Name, InfluxDBHost, InfluxDBPort, WriteSyntax, EnableBatch, QueryMode, UseTLS]
+        ),
+    {Name, ConfigString, parse_and_check(ConfigString, Type, Name)}.
+
+parse_and_check(ConfigString, Type, Name) ->
+    {ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
+    TypeBin = influxdb_type_bin(Type),
+    hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
+    #{<<"bridges">> := #{TypeBin := #{Name := Config}}} = RawConf,
+    Config.
+
+influxdb_type_bin(apiv1) ->
+    <<"influxdb_api_v1">>;
+influxdb_type_bin(apiv2) ->
+    <<"influxdb_api_v2">>.
+
+create_bridge(Config) ->
+    Type = influxdb_type_bin(?config(influxdb_type, Config)),
+    Name = ?config(influxdb_name, Config),
+    InfluxDBConfig = ?config(influxdb_config, Config),
+    emqx_bridge:create(Type, Name, InfluxDBConfig).
+
+delete_bridge(Config) ->
+    Type = influxdb_type_bin(?config(influxdb_type, Config)),
+    Name = ?config(influxdb_name, Config),
+    emqx_bridge:remove(Type, Name).
+
+delete_all_bridges() ->
+    lists:foreach(
+        fun(#{name := Name, type := Type}) ->
+            emqx_bridge:remove(Type, Name)
+        end,
+        emqx_bridge:list()
+    ).
+
+send_message(Config, Payload) ->
+    Name = ?config(influxdb_name, Config),
+    Type = influxdb_type_bin(?config(influxdb_type, Config)),
+    BridgeId = emqx_bridge_resource:bridge_id(Type, Name),
+    emqx_bridge:send_message(BridgeId, Payload).
+
+query_by_clientid(ClientId, Config) ->
+    InfluxDBHost = ?config(influxdb_host, Config),
+    InfluxDBPort = ?config(influxdb_port, Config),
+    EHttpcPoolName = ?config(ehttpc_pool_name, Config),
+    UseTLS = ?config(use_tls, Config),
+    Path = <<"/api/v2/query?org=emqx">>,
+    Scheme =
+        case UseTLS of
+            true -> <<"https://">>;
+            false -> <<"http://">>
+        end,
+    URI = iolist_to_binary([
+        Scheme,
+        list_to_binary(InfluxDBHost),
+        ":",
+        integer_to_binary(InfluxDBPort),
+        Path
+    ]),
+    Query =
+        <<
+            "from(bucket: \"mqtt\")\n"
+            "  |> range(start: -12h)\n"
+            "  |> filter(fn: (r) => r.clientid == \"",
+            ClientId/binary,
+            "\")"
+        >>,
+    Headers = [
+        {"Authorization", "Token abcdefg"},
+        {"Content-Type", "application/json"}
+    ],
+    Body =
+        emqx_json:encode(#{
+            query => Query,
+            dialect => #{
+                header => true,
+                delimiter => <<";">>
+            }
+        }),
+    {ok, 200, _Headers, RawBody0} =
+        ehttpc:request(
+            EHttpcPoolName,
+            post,
+            {URI, Headers, Body},
+            _Timeout = 10_000,
+            _Retry = 0
+        ),
+    RawBody1 = iolist_to_binary(string:replace(RawBody0, <<"\r\n">>, <<"\n">>, all)),
+    {ok, DecodedCSV0} = erl_csv:decode(RawBody1, #{separator => <<$;>>}),
+    DecodedCSV1 = [
+        [Field || Field <- Line, Field =/= <<>>]
+     || Line <- DecodedCSV0,
+        Line =/= [<<>>]
+    ],
+    DecodedCSV2 = csv_lines_to_maps(DecodedCSV1, []),
+    index_by_field(DecodedCSV2).
+
+decode_csv(RawBody) ->
+    Lines =
+        [
+            binary:split(Line, [<<";">>], [global, trim_all])
+         || Line <- binary:split(RawBody, [<<"\r\n">>], [global, trim_all])
+        ],
+    csv_lines_to_maps(Lines, []).
+
+csv_lines_to_maps([Fields, Data | Rest], Acc) ->
+    Map = maps:from_list(lists:zip(Fields, Data)),
+    csv_lines_to_maps(Rest, [Map | Acc]);
+csv_lines_to_maps(_Data, Acc) ->
+    lists:reverse(Acc).
+
+index_by_field(DecodedCSV) ->
+    maps:from_list([{Field, Data} || Data = #{<<"_field">> := Field} <- DecodedCSV]).
+
+assert_persisted_data(ClientId, Expected, PersistedData) ->
+    ClientIdIntKey = <<ClientId/binary, "_int_value">>,
+    maps:foreach(
+        fun
+            (int_value, ExpectedValue) ->
+                ?assertMatch(
+                    #{<<"_value">> := ExpectedValue},
+                    maps:get(ClientIdIntKey, PersistedData)
+                );
+            (Key, ExpectedValue) ->
+                ?assertMatch(
+                    #{<<"_value">> := ExpectedValue},
+                    maps:get(atom_to_binary(Key), PersistedData),
+                    #{expected => ExpectedValue}
+                )
+        end,
+        Expected
+    ),
+    ok.
+
+resource_id(Config) ->
+    Type = influxdb_type_bin(?config(influxdb_type, Config)),
+    Name = ?config(influxdb_name, Config),
+    emqx_bridge_resource:resource_id(Type, Name).
+
+instance_id(Config) ->
+    ResourceId = resource_id(Config),
+    [{_, InstanceId}] = ets:lookup(emqx_resource_manager, {owner, ResourceId}),
+    InstanceId.
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_start_ok(Config) ->
+    QueryMode = ?config(query_mode, Config),
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    ClientId = emqx_guid:to_hexstr(emqx_guid:gen()),
+    Payload = #{
+        int_key => -123,
+        bool => true,
+        float_key => 24.5,
+        uint_key => 123
+    },
+    SentData = #{
+        <<"clientid">> => ClientId,
+        <<"topic">> => atom_to_binary(?FUNCTION_NAME),
+        <<"timestamp">> => erlang:system_time(nanosecond),
+        <<"payload">> => Payload
+    },
+    ?check_trace(
+        begin
+            ?assertEqual(ok, send_message(Config, SentData)),
+            case QueryMode of
+                async -> ct:sleep(500);
+                sync -> ok
+            end,
+            PersistedData = query_by_clientid(ClientId, Config),
+            Expected = #{
+                bool => <<"true">>,
+                int_value => <<"-123">>,
+                uint_value => <<"123">>,
+                float_value => <<"24.5">>,
+                payload => emqx_json:encode(Payload)
+            },
+            assert_persisted_data(ClientId, Expected, PersistedData),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(influxdb_connector_send_query, Trace0),
+            ?assertMatch([#{points := [_]}], Trace),
+            [#{points := [Point]}] = Trace,
+            ct:pal("sent point: ~p", [Point]),
+            ?assertMatch(
+                #{
+                    fields := #{},
+                    measurement := <<_/binary>>,
+                    tags := #{},
+                    timestamp := TS
+                } when is_integer(TS),
+                Point
+            ),
+            #{fields := Fields} = Point,
+            ?assert(lists:all(fun is_binary/1, maps:keys(Fields))),
+            ?assertNot(maps:is_key(<<"undefined">>, Fields)),
+            ?assertNot(maps:is_key(<<"undef_value">>, Fields)),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_start_already_started(Config) ->
+    Type = influxdb_type_bin(?config(influxdb_type, Config)),
+    Name = ?config(influxdb_name, Config),
+    InfluxDBConfigString = ?config(influxdb_config_string, Config),
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    InstanceId = instance_id(Config),
+    TypeAtom = binary_to_atom(Type),
+    NameAtom = binary_to_atom(Name),
+    {ok, #{bridges := #{TypeAtom := #{NameAtom := InfluxDBConfigMap}}}} = emqx_hocon:check(
+        emqx_bridge_schema, InfluxDBConfigString
+    ),
+    ?check_trace(
+        emqx_ee_connector_influxdb:on_start(InstanceId, InfluxDBConfigMap),
+        fun(Result, Trace) ->
+            ?assertMatch({ok, _}, Result),
+            ?assertMatch([_], ?of_kind(influxdb_connector_start_already_started, Trace)),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_start_ok_timestamp_write_syntax(Config) ->
+    InfluxDBType = ?config(influxdb_type, Config),
+    InfluxDBName = ?config(influxdb_name, Config),
+    InfluxDBConfigString0 = ?config(influxdb_config_string, Config),
+    InfluxDBTypeCfg =
+        case InfluxDBType of
+            apiv1 -> "influxdb_api_v1";
+            apiv2 -> "influxdb_api_v2"
+        end,
+    WriteSyntax =
+        %% N.B.: this single space characters are relevant
+        <<"${topic},clientid=${clientid}", " ", "payload=${payload},",
+            "${clientid}_int_value=${payload.int_key}i,",
+            "uint_value=${payload.uint_key}u,"
+            "bool=${payload.bool}", " ", "${timestamp}">>,
+    %% append this to override the config
+    InfluxDBConfigString1 =
+        io_lib:format(
+            "bridges.~s.~s {\n"
+            "  write_syntax = \"~s\"\n"
+            "}\n",
+            [InfluxDBTypeCfg, InfluxDBName, WriteSyntax]
+        ),
+    InfluxDBConfig1 = parse_and_check(
+        InfluxDBConfigString0 ++ InfluxDBConfigString1,
+        InfluxDBType,
+        InfluxDBName
+    ),
+    Config1 = [{influxdb_config, InfluxDBConfig1} | Config],
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config1)
+    ),
+    ok.
+
+t_start_ok_no_subject_tags_write_syntax(Config) ->
+    InfluxDBType = ?config(influxdb_type, Config),
+    InfluxDBName = ?config(influxdb_name, Config),
+    InfluxDBConfigString0 = ?config(influxdb_config_string, Config),
+    InfluxDBTypeCfg =
+        case InfluxDBType of
+            apiv1 -> "influxdb_api_v1";
+            apiv2 -> "influxdb_api_v2"
+        end,
+    WriteSyntax =
+        %% N.B.: this single space characters are relevant
+        <<"${topic}", " ", "payload=${payload},", "${clientid}_int_value=${payload.int_key}i,",
+            "uint_value=${payload.uint_key}u,"
+            "bool=${payload.bool}", " ", "${timestamp}">>,
+    %% append this to override the config
+    InfluxDBConfigString1 =
+        io_lib:format(
+            "bridges.~s.~s {\n"
+            "  write_syntax = \"~s\"\n"
+            "}\n",
+            [InfluxDBTypeCfg, InfluxDBName, WriteSyntax]
+        ),
+    InfluxDBConfig1 = parse_and_check(
+        InfluxDBConfigString0 ++ InfluxDBConfigString1,
+        InfluxDBType,
+        InfluxDBName
+    ),
+    Config1 = [{influxdb_config, InfluxDBConfig1} | Config],
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config1)
+    ),
+    ok.
+
+t_boolean_variants(Config) ->
+    QueryMode = ?config(query_mode, Config),
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    BoolVariants = #{
+        true => true,
+        false => false,
+        <<"t">> => true,
+        <<"f">> => false,
+        <<"T">> => true,
+        <<"F">> => false,
+        <<"TRUE">> => true,
+        <<"FALSE">> => false,
+        <<"True">> => true,
+        <<"False">> => false
+    },
+    maps:foreach(
+        fun(BoolVariant, Translation) ->
+            ClientId = emqx_guid:to_hexstr(emqx_guid:gen()),
+            Payload = #{
+                int_key => -123,
+                bool => BoolVariant,
+                uint_key => 123
+            },
+            SentData = #{
+                <<"clientid">> => ClientId,
+                <<"topic">> => atom_to_binary(?FUNCTION_NAME),
+                <<"timestamp">> => erlang:system_time(nanosecond),
+                <<"payload">> => Payload
+            },
+            ?assertEqual(ok, send_message(Config, SentData)),
+            case QueryMode of
+                async -> ct:sleep(500);
+                sync -> ok
+            end,
+            PersistedData = query_by_clientid(ClientId, Config),
+            Expected = #{
+                bool => atom_to_binary(Translation),
+                int_value => <<"-123">>,
+                uint_value => <<"123">>,
+                payload => emqx_json:encode(Payload)
+            },
+            assert_persisted_data(ClientId, Expected, PersistedData),
+            ok
+        end,
+        BoolVariants
+    ),
+    ok.
+
+t_bad_timestamp(Config) ->
+    InfluxDBType = ?config(influxdb_type, Config),
+    InfluxDBName = ?config(influxdb_name, Config),
+    QueryMode = ?config(query_mode, Config),
+    EnableBatch = ?config(enable_batch, Config),
+    InfluxDBConfigString0 = ?config(influxdb_config_string, Config),
+    InfluxDBTypeCfg =
+        case InfluxDBType of
+            apiv1 -> "influxdb_api_v1";
+            apiv2 -> "influxdb_api_v2"
+        end,
+    WriteSyntax =
+        %% N.B.: this single space characters are relevant
+        <<"${topic}", " ", "payload=${payload},", "${clientid}_int_value=${payload.int_key}i,",
+            "uint_value=${payload.uint_key}u,"
+            "bool=${payload.bool}", " ", "bad_timestamp">>,
+    %% append this to override the config
+    InfluxDBConfigString1 =
+        io_lib:format(
+            "bridges.~s.~s {\n"
+            "  write_syntax = \"~s\"\n"
+            "}\n",
+            [InfluxDBTypeCfg, InfluxDBName, WriteSyntax]
+        ),
+    InfluxDBConfig1 = parse_and_check(
+        InfluxDBConfigString0 ++ InfluxDBConfigString1,
+        InfluxDBType,
+        InfluxDBName
+    ),
+    Config1 = [{influxdb_config, InfluxDBConfig1} | Config],
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config1)
+    ),
+    ClientId = emqx_guid:to_hexstr(emqx_guid:gen()),
+    Payload = #{
+        int_key => -123,
+        bool => false,
+        uint_key => 123
+    },
+    SentData = #{
+        <<"clientid">> => ClientId,
+        <<"topic">> => atom_to_binary(?FUNCTION_NAME),
+        <<"timestamp">> => erlang:system_time(nanosecond),
+        <<"payload">> => Payload
+    },
+    ?check_trace(
+        ?wait_async_action(
+            send_message(Config1, SentData),
+            #{?snk_kind := influxdb_connector_send_query_error},
+            10_000
+        ),
+        fun(Result, Trace) ->
+            ?assertMatch({_, {ok, _}}, Result),
+            {Return, {ok, _}} = Result,
+            case {QueryMode, EnableBatch} of
+                {async, true} ->
+                    ?assertEqual(ok, Return),
+                    ?assertMatch(
+                        [#{error := points_trans_failed}],
+                        ?of_kind(influxdb_connector_send_query_error, Trace)
+                    );
+                {async, false} ->
+                    ?assertEqual(ok, Return),
+                    ?assertMatch(
+                        [#{error := [{error, {bad_timestamp, [<<"bad_timestamp">>]}}]}],
+                        ?of_kind(influxdb_connector_send_query_error, Trace)
+                    );
+                {sync, false} ->
+                    ?assertEqual(
+                        {error, [{error, {bad_timestamp, [<<"bad_timestamp">>]}}]}, Return
+                    );
+                {sync, true} ->
+                    ?assertEqual({error, points_trans_failed}, Return)
+            end,
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_get_status(Config) ->
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyName = ?config(proxy_name, Config),
+    {ok, _} = create_bridge(Config),
+    ResourceId = resource_id(Config),
+    ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)),
+    emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+        ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId))
+    end),
+    ok.
+
+t_create_disconnected(Config) ->
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyName = ?config(proxy_name, Config),
+    ?check_trace(
+        emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+            ?assertMatch({ok, _}, create_bridge(Config))
+        end),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{error := influxdb_client_not_alive}],
+                ?of_kind(influxdb_connector_start_failed, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_start_error(Config) ->
+    %% simulate client start error
+    ?check_trace(
+        emqx_common_test_helpers:with_mock(
+            influxdb,
+            start_client,
+            fun(_Config) -> {error, some_error} end,
+            fun() ->
+                ?wait_async_action(
+                    ?assertMatch({ok, _}, create_bridge(Config)),
+                    #{?snk_kind := influxdb_connector_start_failed},
+                    10_000
+                )
+            end
+        ),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{error := some_error}],
+                ?of_kind(influxdb_connector_start_failed, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_start_exception(Config) ->
+    %% simulate client start exception
+    ?check_trace(
+        emqx_common_test_helpers:with_mock(
+            influxdb,
+            start_client,
+            fun(_Config) -> error(boom) end,
+            fun() ->
+                ?wait_async_action(
+                    ?assertMatch({ok, _}, create_bridge(Config)),
+                    #{?snk_kind := influxdb_connector_start_exception},
+                    10_000
+                )
+            end
+        ),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{error := {error, boom}}],
+                ?of_kind(influxdb_connector_start_exception, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_write_failure(Config) ->
+    ProxyName = ?config(proxy_name, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    QueryMode = ?config(query_mode, Config),
+    {ok, _} = create_bridge(Config),
+    ClientId = emqx_guid:to_hexstr(emqx_guid:gen()),
+    Payload = #{
+        int_key => -123,
+        bool => true,
+        float_key => 24.5,
+        uint_key => 123
+    },
+    SentData = #{
+        <<"clientid">> => ClientId,
+        <<"topic">> => atom_to_binary(?FUNCTION_NAME),
+        <<"timestamp">> => erlang:system_time(nanosecond),
+        <<"payload">> => Payload
+    },
+    ?check_trace(
+        emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+            send_message(Config, SentData)
+        end),
+        fun(Result, _Trace) ->
+            case QueryMode of
+                sync ->
+                    ?assert(
+                        {error, {error, {closed, "The connection was lost."}}} =:= Result orelse
+                            {error, {error, closed}} =:= Result orelse
+                            {error, {error, econnrefused}} =:= Result,
+                        #{got => Result}
+                    );
+                async ->
+                    ?assertEqual(ok, Result)
+            end,
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 3f86792ff..36b2ec44d 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -9,6 +9,7 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
@@ -51,8 +52,16 @@ on_stop(_InstId, #{client := Client}) ->
 on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, client := Client}) ->
     case data_to_points(Data, SyntaxLines) of
         {ok, Points} ->
+            ?tp(
+                influxdb_connector_send_query,
+                #{points => Points, batch => false, mode => sync}
+            ),
             do_query(InstId, Client, Points);
         {error, ErrorPoints} = Err ->
+            ?tp(
+                influxdb_connector_send_query_error,
+                #{batch => false, mode => sync, error => ErrorPoints}
+            ),
             log_error_points(InstId, ErrorPoints),
             Err
     end.
@@ -62,8 +71,16 @@ on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, c
 on_batch_query(InstId, BatchData, _State = #{write_syntax := SyntaxLines, client := Client}) ->
     case parse_batch_data(InstId, BatchData, SyntaxLines) of
         {ok, Points} ->
+            ?tp(
+                influxdb_connector_send_query,
+                #{points => Points, batch => true, mode => sync}
+            ),
             do_query(InstId, Client, Points);
         {error, Reason} ->
+            ?tp(
+                influxdb_connector_send_query_error,
+                #{batch => true, mode => sync, error => Reason}
+            ),
             {error, Reason}
     end.
 
@@ -75,8 +92,16 @@ on_query_async(
 ) ->
     case data_to_points(Data, SyntaxLines) of
         {ok, Points} ->
+            ?tp(
+                influxdb_connector_send_query,
+                #{points => Points, batch => false, mode => async}
+            ),
             do_async_query(InstId, Client, Points, {ReplayFun, Args});
         {error, ErrorPoints} = Err ->
+            ?tp(
+                influxdb_connector_send_query_error,
+                #{batch => false, mode => async, error => ErrorPoints}
+            ),
             log_error_points(InstId, ErrorPoints),
             Err
     end.
@@ -89,8 +114,16 @@ on_batch_query_async(
 ) ->
     case parse_batch_data(InstId, BatchData, SyntaxLines) of
         {ok, Points} ->
+            ?tp(
+                influxdb_connector_send_query,
+                #{points => Points, batch => true, mode => async}
+            ),
             do_async_query(InstId, Client, Points, {ReplayFun, Args});
         {error, Reason} ->
+            ?tp(
+                influxdb_connector_send_query_error,
+                #{batch => true, mode => async, error => Reason}
+            ),
             {error, Reason}
     end.
 
@@ -163,6 +196,7 @@ start_client(InstId, Config) ->
         do_start_client(InstId, ClientConfig, Config)
     catch
         E:R:S ->
+            ?tp(influxdb_connector_start_exception, #{error => {E, R}}),
             ?SLOG(error, #{
                 msg => "start influxdb connector error",
                 connector => InstId,
@@ -196,6 +230,7 @@ do_start_client(
                     }),
                     {ok, State};
                 false ->
+                    ?tp(influxdb_connector_start_failed, #{error => influxdb_client_not_alive}),
                     ?SLOG(error, #{
                         msg => "starting influxdb connector failed",
                         connector => InstId,
@@ -205,14 +240,16 @@ do_start_client(
                     {error, influxdb_client_not_alive}
             end;
         {error, {already_started, Client0}} ->
+            ?tp(influxdb_connector_start_already_started, #{}),
             ?SLOG(info, #{
-                msg => "starting influxdb connector,find already started client",
+                msg => "restarting influxdb connector, found already started client",
                 connector => InstId,
                 old_client => Client0
             }),
             _ = influxdb:stop_client(Client0),
             do_start_client(InstId, ClientConfig, Config);
         {error, Reason} ->
+            ?tp(influxdb_connector_start_failed, #{error => Reason}),
             ?SLOG(error, #{
                 msg => "starting influxdb connector failed",
                 connector => InstId,
@@ -235,7 +272,7 @@ client_config(
         {precision, atom_to_binary(maps:get(precision, Config, ms), utf8)}
     ] ++ protocol_config(Config).
 
-%% api v2 config
+%% api v1 config
 protocol_config(#{
     username := Username,
     password := Password,
@@ -249,7 +286,7 @@ protocol_config(#{
         {password, str(Password)},
         {database, str(DB)}
     ] ++ ssl_config(SSL);
-%% api v1 config
+%% api v2 config
 protocol_config(#{
     bucket := Bucket,
     org := Org,
@@ -290,6 +327,7 @@ do_query(InstId, Client, Points) ->
                 points => Points
             });
         {error, Reason} = Err ->
+            ?tp(influxdb_connector_do_query_failure, #{error => Reason}),
             ?SLOG(error, #{
                 msg => "influxdb write point failed",
                 connector => InstId,
@@ -370,6 +408,14 @@ parse_batch_data(InstId, BatchData, SyntaxLines) ->
             {error, points_trans_failed}
     end.
 
+-spec data_to_points(map(), [
+    #{
+        fields := [{binary(), binary()}],
+        measurement := binary(),
+        tags := [{binary(), binary()}],
+        timestamp := binary()
+    }
+]) -> {ok, [map()]} | {error, term()}.
 data_to_points(Data, SyntaxLines) ->
     lines_to_points(Data, SyntaxLines, [], []).
 
@@ -416,17 +462,18 @@ lines_to_points(
     end.
 
 maps_config_to_data(K, V, {Data, Res}) ->
-    KTransOptions = #{return => full_binary},
+    KTransOptions = #{return => rawlist, var_trans => fun key_filter/1},
     VTransOptions = #{return => rawlist, var_trans => fun data_filter/1},
-    NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions),
+    NK0 = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions),
     NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions),
-    case {NK, NV} of
+    case {NK0, NV} of
         {[undefined], _} ->
             {Data, Res};
         %% undefined value in normal format [undefined] or int/uint format [undefined, <<"i">>]
         {_, [undefined | _]} ->
             {Data, Res};
         _ ->
+            NK = list_to_binary(NK0),
             {Data, Res#{NK => value_type(NV)}}
     end.
 
@@ -438,6 +485,8 @@ value_type([UInt, <<"u">>]) when
     is_integer(UInt)
 ->
     {uint, UInt};
+value_type([Float]) when is_float(Float) ->
+    Float;
 value_type([<<"t">>]) ->
     't';
 value_type([<<"T">>]) ->
@@ -461,6 +510,9 @@ value_type([<<"False">>]) ->
 value_type(Val) ->
     Val.
 
+key_filter(undefined) -> undefined;
+key_filter(Value) -> emqx_plugin_libs_rule:bin(Value).
+
 data_filter(undefined) -> undefined;
 data_filter(Int) when is_integer(Int) -> Int;
 data_filter(Number) when is_number(Number) -> Number;
@@ -500,3 +552,56 @@ str(B) when is_binary(B) ->
     binary_to_list(B);
 str(S) when is_list(S) ->
     S.
+
+%%===================================================================
+%% eunit tests
+%%===================================================================
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+to_server_raw_test_() ->
+    [
+        ?_assertEqual(
+            {"foobar", 1234},
+            to_server_raw(<<"http://foobar:1234">>)
+        ),
+        ?_assertEqual(
+            {"foobar", 1234},
+            to_server_raw(<<"https://foobar:1234">>)
+        ),
+        ?_assertEqual(
+            {"foobar", 1234},
+            to_server_raw(<<"foobar:1234">>)
+        )
+    ].
+
+%% for coverage
+desc_test_() ->
+    [
+        ?_assertMatch(
+            {desc, _, _},
+            desc(common)
+        ),
+        ?_assertMatch(
+            {desc, _, _},
+            desc(influxdb_udp)
+        ),
+        ?_assertMatch(
+            {desc, _, _},
+            desc(influxdb_api_v1)
+        ),
+        ?_assertMatch(
+            {desc, _, _},
+            desc(influxdb_api_v2)
+        ),
+        ?_assertMatch(
+            {desc, _, _},
+            server(desc)
+        ),
+        ?_assertMatch(
+            connector_influxdb,
+            namespace()
+        )
+    ].
+-endif.
diff --git a/rebar.config.erl b/rebar.config.erl
index f745b5cca..bbfdfe4a8 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -150,7 +150,8 @@ test_deps() ->
         {bbmustache, "1.10.0"},
         {meck, "0.9.2"},
         {proper, "1.4.0"},
-        {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}}
+        {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}},
+        {erl_csv, "0.2.0"}
     ].
 
 common_compile_opts(Vsn) ->
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 45d32767c..38511ec3e 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -92,6 +92,11 @@ for dep in ${CT_DEPS}; do
         erlang24)
             FILES+=( '.ci/docker-compose-file/docker-compose.yaml' )
             ;;
+        influxdb)
+          FILES+=( '.ci/docker-compose-file/docker-compose-toxiproxy.yaml'
+                   '.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml'
+                   '.ci/docker-compose-file/docker-compose-influxdb-tls.yaml' )
+            ;;
         mongo)
             FILES+=( '.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml'
                      '.ci/docker-compose-file/docker-compose-mongo-single-tls.yaml' )

From b673bef0275538114cfba15c81d45d70a63070ca Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 11 Nov 2022 10:02:02 -0300
Subject: [PATCH 189/232] style: indent script

---
 scripts/ct/run.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 38511ec3e..c6d05cf3b 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -93,9 +93,9 @@ for dep in ${CT_DEPS}; do
             FILES+=( '.ci/docker-compose-file/docker-compose.yaml' )
             ;;
         influxdb)
-          FILES+=( '.ci/docker-compose-file/docker-compose-toxiproxy.yaml'
-                   '.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml'
-                   '.ci/docker-compose-file/docker-compose-influxdb-tls.yaml' )
+            FILES+=( '.ci/docker-compose-file/docker-compose-toxiproxy.yaml'
+                     '.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml'
+                     '.ci/docker-compose-file/docker-compose-influxdb-tls.yaml' )
             ;;
         mongo)
             FILES+=( '.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml'

From 0f9cc0d93f086e116bb5778c8517086df04854fd Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 11 Nov 2022 10:03:30 -0300
Subject: [PATCH 190/232] test(refactor): stop snabbkaffe on every test

---
 .../emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
index a05a26e29..d22c6abc0 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
@@ -232,6 +232,7 @@ init_per_testcase(_Testcase, Config) ->
 end_per_testcase(_Testcase, Config) ->
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
+    ok = snabbkaffe:stop(),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
     %% catch clear_db(Config),
     %% delete_bridge(Config),
@@ -509,7 +510,6 @@ t_start_ok(Config) ->
             ok
         end
     ),
-    ok = snabbkaffe:stop(),
     ok.
 
 t_start_already_started(Config) ->
@@ -534,7 +534,6 @@ t_start_already_started(Config) ->
             ok
         end
     ),
-    ok = snabbkaffe:stop(),
     ok.
 
 t_start_ok_timestamp_write_syntax(Config) ->
@@ -735,7 +734,6 @@ t_bad_timestamp(Config) ->
             ok
         end
     ),
-    ok = snabbkaffe:stop(),
     ok.
 
 t_get_status(Config) ->
@@ -766,7 +764,6 @@ t_create_disconnected(Config) ->
             ok
         end
     ),
-    ok = snabbkaffe:stop(),
     ok.
 
 t_start_error(Config) ->
@@ -792,7 +789,6 @@ t_start_error(Config) ->
             ok
         end
     ),
-    ok = snabbkaffe:stop(),
     ok.
 
 t_start_exception(Config) ->
@@ -818,7 +814,6 @@ t_start_exception(Config) ->
             ok
         end
     ),
-    ok = snabbkaffe:stop(),
     ok.
 
 t_write_failure(Config) ->
@@ -859,5 +854,4 @@ t_write_failure(Config) ->
             ok
         end
     ),
-    ok = snabbkaffe:stop(),
     ok.

From 2ae0125562d59fc88bc87bd79fb29e3a14c66cab Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 16 Nov 2022 10:57:13 -0300
Subject: [PATCH 191/232] test(refactor): encode toxiproxy payloads with
 emqx_json

---
 apps/emqx/test/emqx_common_test_helpers.erl | 40 +++++++++++++--------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 196546b7b..8ab3b3ace 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -837,12 +837,13 @@ switch_proxy(Switch, Name, ProxyHost, ProxyPort) ->
     Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name,
     Body =
         case Switch of
-            off -> <<"{\"enabled\":false}">>;
-            on -> <<"{\"enabled\":true}">>
+            off -> #{<<"enabled">> => false};
+            on -> #{<<"enabled">> => true}
         end,
+    BodyBin = emqx_json:encode(Body),
     {ok, {{_, 200, _}, _, _}} = httpc:request(
         post,
-        {Url, [], "application/json", Body},
+        {Url, [], "application/json", BodyBin},
         [],
         [{body_format, binary}]
     ).
@@ -852,14 +853,17 @@ timeout_proxy(on, Name, ProxyHost, ProxyPort) ->
         "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name ++
             "/toxics",
     NameBin = list_to_binary(Name),
-    Body =
-        <<"{\"name\":\"", NameBin/binary,
-            "_timeout\",\"type\":\"timeout\","
-            "\"stream\":\"upstream\",\"toxicity\":1.0,"
-            "\"attributes\":{\"timeout\":0}}">>,
+    Body = #{
+        <<"name">> => <<NameBin/binary, "_timeout">>,
+        <<"type">> => <<"timeout">>,
+        <<"stream">> => <<"upstream">>,
+        <<"toxicity">> => 1.0,
+        <<"attributes">> => #{<<"timeout">> => 0}
+    },
+    BodyBin = emqx_json:encode(Body),
     {ok, {{_, 200, _}, _, _}} = httpc:request(
         post,
-        {Url, [], "application/json", Body},
+        {Url, [], "application/json", BodyBin},
         [],
         [{body_format, binary}]
     );
@@ -881,14 +885,20 @@ latency_up_proxy(on, Name, ProxyHost, ProxyPort) ->
         "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/" ++ Name ++
             "/toxics",
     NameBin = list_to_binary(Name),
-    Body =
-        <<"{\"name\":\"", NameBin/binary,
-            "_latency_up\",\"type\":\"latency\","
-            "\"stream\":\"upstream\",\"toxicity\":1.0,"
-            "\"attributes\":{\"latency\":20000,\"jitter\":3000}}">>,
+    Body = #{
+        <<"name">> => <<NameBin/binary, "_latency_up">>,
+        <<"type">> => <<"latency">>,
+        <<"stream">> => <<"upstream">>,
+        <<"toxicity">> => 1.0,
+        <<"attributes">> => #{
+            <<"latency">> => 20_000,
+            <<"jitter">> => 3_000
+        }
+    },
+    BodyBin = emqx_json:encode(Body),
     {ok, {{_, 200, _}, _, _}} = httpc:request(
         post,
-        {Url, [], "application/json", Body},
+        {Url, [], "application/json", BodyBin},
         [],
         [{body_format, binary}]
     );

From c940b901f5f03128198082e8f2cb90dc30f4dbc5 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 16 Nov 2022 16:26:43 +0100
Subject: [PATCH 192/232] chore: fix app versions

---
 apps/emqx/src/emqx.app.src                             | 2 +-
 apps/emqx_authn/src/emqx_authn.app.src                 | 2 +-
 apps/emqx_authz/src/emqx_authz.app.src                 | 2 +-
 apps/emqx_bridge/src/emqx_bridge.app.src               | 4 ++--
 apps/emqx_conf/src/emqx_conf.app.src                   | 2 +-
 apps/emqx_rule_engine/src/emqx_rule_engine.app.src     | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src       | 2 +-
 lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src           | 2 +-
 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +-
 scripts/apps-version-check.sh                          | 4 ++--
 10 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src
index 5d2d8eb2c..3d1fe32d3 100644
--- a/apps/emqx/src/emqx.app.src
+++ b/apps/emqx/src/emqx.app.src
@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.10"},
+    {vsn, "5.0.11"},
     {modules, []},
     {registered, []},
     {applications, [
diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index fe4a2888d..038d2601a 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.8"},
+    {vsn, "0.1.9"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},
diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index 33d9a7e41..e6913b43c 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src
index 7890853e4..fcaf9394f 100644
--- a/apps/emqx_bridge/src/emqx_bridge.app.src
+++ b/apps/emqx_bridge/src/emqx_bridge.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
-    {description, "An OTP application"},
-    {vsn, "0.1.4"},
+    {description, "EMQX bridges"},
+    {vsn, "0.1.5"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [
diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src
index 2b6efe639..c57664ca7 100644
--- a/apps/emqx_conf/src/emqx_conf.app.src
+++ b/apps/emqx_conf/src/emqx_conf.app.src
@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.1.6"},
+    {vsn, "0.1.7"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 6bb9ad010..6419e4184 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.3"},
+    {vsn, "5.0.4"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt]},
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index 7759ef2a2..2e6458682 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -1,5 +1,5 @@
 {application, emqx_ee_bridge, [
-    {vsn, "0.1.1"},
+    {vsn, "0.1.0"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
index 771fdcb27..324e7e308 100644
--- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
+++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_conf, [
     {description, "EMQX Enterprise Edition configuration schema"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.0"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index b2aed3624..c1b86d20b 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,5 +1,5 @@
 {application, emqx_ee_connector, [
-    {vsn, "0.1.1"},
+    {vsn, "0.1.0"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh
index 917fb5093..95e71f128 100755
--- a/scripts/apps-version-check.sh
+++ b/scripts/apps-version-check.sh
@@ -28,7 +28,7 @@ for app in ${APPS}; do
     now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
 
     if [ "$old_app_version" = 'not_found' ]; then
-        echo "$src_file is newly added"
+        echo "IGNORE: $src_file is newly added"
         true
     elif [ "$old_app_version" = "$now_app_version" ]; then
         changed_lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \
@@ -38,7 +38,7 @@ for app in ${APPS}; do
                              -- "$app_path/priv" \
                              -- "$app_path/c_src" | wc -l ) "
         if [ "$changed_lines" -gt 0 ]; then
-            echo "$src_file needs a vsn bump"
+            echo "ERROR: $src_file needs a vsn bump"
             bad_app_count=$(( bad_app_count + 1))
         fi
     else

From 7305923d122fb4503ef0d24d1a7c3e02926b5d0f Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 16 Nov 2022 18:25:42 +0100
Subject: [PATCH 193/232] fix: bridge name parser should not leak atom

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      |  5 +--
 apps/emqx_bridge/src/emqx_bridge_resource.erl | 40 ++++++++++++++++++-
 .../kafka/emqx_bridge_impl_kafka_producer.erl |  4 +-
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index d513fd38d..6b5e307d8 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -48,12 +48,11 @@
         {BridgeType, BridgeName} ->
             EXPR
     catch
-        error:{invalid_bridge_id, Id0} ->
+        throw:{invalid_bridge_id, Reason} ->
             {400,
                 error_msg(
                     'INVALID_ID',
-                    <<"invalid_bridge_id: ", Id0/binary,
-                        ". Bridge Ids must be of format {type}:{name}">>
+                    <<"Invalid bride ID, ", Reason/binary>>
                 )}
     end
 ).
diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index bb5f5cc54..46722bc59 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -74,8 +74,44 @@ bridge_id(BridgeType, BridgeName) ->
 -spec parse_bridge_id(list() | binary() | atom()) -> {atom(), binary()}.
 parse_bridge_id(BridgeId) ->
     case string:split(bin(BridgeId), ":", all) of
-        [Type, Name] -> {binary_to_atom(Type, utf8), Name};
-        _ -> error({invalid_bridge_id, BridgeId})
+        [Type, Name] ->
+            {to_type_atom(Type), validate_name(Name)};
+        _ ->
+            invalid_bridge_id(
+                <<"should be of forst {type}:{name}, but got ", BridgeId/binary>>
+            )
+    end.
+
+validate_name(Name0) ->
+    Name = unicode:characters_to_list(Name0, utf8),
+    case is_list(Name) andalso Name =/= [] of
+        true ->
+            case lists:all(fun is_id_char/1, Name) of
+                true ->
+                    Name0;
+                false ->
+                    invalid_bridge_id(<<"bad name: ", Name0/binary>>)
+            end;
+        false ->
+            invalid_bridge_id(<<"only 0-9a-zA-Z_-. is allowed in name: ", Name0/binary>>)
+    end.
+
+invalid_bridge_id(Reason) -> throw({?FUNCTION_NAME, Reason}).
+
+is_id_char(C) when C >= $0 andalso C =< $9 -> true;
+is_id_char(C) when C >= $a andalso C =< $z -> true;
+is_id_char(C) when C >= $A andalso C =< $Z -> true;
+is_id_char($_) -> true;
+is_id_char($-) -> true;
+is_id_char($.) -> true;
+is_id_char(_) -> false.
+
+to_type_atom(Type) ->
+    try
+        erlang:binary_to_existing_atom(Type, utf8)
+    catch
+        _:_ ->
+            invalid_bridge_id(<<"unknown type: ", Type/binary>>)
     end.
 
 reset_metrics(ResourceId) ->
diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
index aceafd6d1..0eeb9db0c 100644
--- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -266,7 +266,9 @@ make_client_id(BridgeName) ->
 make_producer_name(BridgeName) when is_atom(BridgeName) ->
     make_producer_name(atom_to_list(BridgeName));
 make_producer_name(BridgeName) ->
-    list_to_atom("kafka_producer_" ++ BridgeName).
+    %% Woff needs atom for ets table name registration
+    %% The assumption here is bridge is not often re-created
+    binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName])).
 
 with_log_at_error(Fun, Log) ->
     try

From 7c8ed59edd6685372bab5daae3056eb27c3f63c5 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 16 Nov 2022 18:30:49 +0100
Subject: [PATCH 194/232] ci: add back the profile matrix for elixir_release
 build

---
 .github/workflows/elixir_release.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml
index 7301073a1..b93e6a675 100644
--- a/.github/workflows/elixir_release.yml
+++ b/.github/workflows/elixir_release.yml
@@ -12,8 +12,12 @@ on:
 jobs:
   elixir_release_build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        profile:
+          - emqx
+          - emqx-enterprise
     container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3

From e5ced076653a7db9dc472fbf61dd924580b9129a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 16 Nov 2022 19:07:51 +0100
Subject: [PATCH 195/232] fix: add no_return function spec to make dialyzer
 happy

---
 apps/emqx_bridge/src/emqx_bridge_resource.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index 46722bc59..ad35485ed 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -96,6 +96,7 @@ validate_name(Name0) ->
             invalid_bridge_id(<<"only 0-9a-zA-Z_-. is allowed in name: ", Name0/binary>>)
     end.
 
+-spec invalid_bridge_id(binary()) -> no_return().
 invalid_bridge_id(Reason) -> throw({?FUNCTION_NAME, Reason}).
 
 is_id_char(C) when C >= $0 andalso C =< $9 -> true;

From 95f3df9a1078cb0c258f28a16a819069aa62487b Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 16 Nov 2022 20:36:16 +0100
Subject: [PATCH 196/232] ci: ensure docker-compose up erlang container with
 root when test Kafka

---
 .github/workflows/run_test_cases.yaml |  2 +-
 scripts/ct/run.sh                     | 28 ++++++++++++++++++---------
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 03030908c..d1794f2fb 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -15,7 +15,7 @@ on:
 
 jobs:
     prepare:
-        runs-on: ubuntu-20.04
+        runs-on: aws-amd64
         # prepare source with any OTP version, no need for a matrix
         container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
         outputs:
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 78b211844..145d915d1 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -115,6 +115,9 @@ for dep in ${CT_DEPS}; do
                      '.ci/docker-compose-file/docker-compose-pgsql-tls.yaml' )
             ;;
         kafka)
+            # Kafka container generates root owned ssl files
+            # the files are shared with EMQX (with a docker volume)
+            NEED_ROOT=yes
             FILES+=( '.ci/docker-compose-file/docker-compose-kafka.yaml' )
             ;;
         *)
@@ -130,13 +133,19 @@ for file in "${FILES[@]}"; do
     F_OPTIONS="$F_OPTIONS -f $file"
 done
 
-# Passing $UID to docker-compose to be used in erlang container
-# as owner of the main process to avoid git repo permissions issue.
-# Permissions issue happens because we are mounting local filesystem
-# where files are owned by $UID to docker container where it's using
-# root (UID=0) by default, and git is not happy about it.
+if [[ "${NEED_ROOT:-}" == 'yes' ]]; then
+    export UID_GID='root:root'
+else
+    # Passing $UID to docker-compose to be used in erlang container
+    # as owner of the main process to avoid git repo permissions issue.
+    # Permissions issue happens because we are mounting local filesystem
+    # where files are owned by $UID to docker container where it's using
+    # root (UID=0) by default, and git is not happy about it.
+    export UID_GID="$UID:$UID"
+fi
+
 # shellcheck disable=2086 # no quotes for F_OPTIONS
-UID_GID="$UID:$UID" docker-compose $F_OPTIONS up -d --build
+docker-compose $F_OPTIONS up -d --build
 
 # /emqx is where the source dir is mounted to the Erlang container
 # in .ci/docker-compose-file/docker-compose.yaml
@@ -145,10 +154,11 @@ if [[ -t 1 ]]; then
     TTY='-t'
 fi
 
+echo "Fixing file owners and permissions for $UID_GID"
 # rebar and hex cache directory need to be writable by $UID
-docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir /.cache && chown $UID:$UID /.cache"
+docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $UID_GID /.cache && chown -R $UID_GID /emqx"
 # need to initialize .erlang.cookie manually here because / is not writable by $UID
-docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID:$UID /.erlang.cookie && chmod 0400 /.erlang.cookie"
+docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID_GID /.erlang.cookie && chmod 0400 /.erlang.cookie"
 
 if [ "$ONLY_UP" = 'yes' ]; then
     exit 0
@@ -166,7 +176,7 @@ else
         exit $RESULT
     else
         # shellcheck disable=2086 # no quotes for F_OPTIONS
-        UID_GID="$UID:$UID" docker-compose $F_OPTIONS down
+        docker-compose $F_OPTIONS down
         exit $RESULT
     fi
 fi

From c6b32c27732415442cd0978683965c0facaafa93 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 18 Nov 2022 12:03:37 +0100
Subject: [PATCH 197/232] ci: ensure PROFILE for ct runs

---
 scripts/ct/run.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 145d915d1..24564b7ae 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -75,7 +75,7 @@ case "${WHICH_APP}" in
         export PROFILE='emqx-enterprise'
         ;;
     *)
-        true
+        export PROFILE="${PROFILE:-emqx}"
         ;;
 esac
 
@@ -167,10 +167,10 @@ fi
 if [ "$ATTACH" = 'yes' ]; then
     docker exec -it "$ERLANG_CONTAINER" bash
 elif [ "$CONSOLE" = 'yes' ]; then
-    docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
+    docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
 else
     set +e
-    docker exec -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
+    docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
     RESULT=$?
     if [ "$KEEP_UP" = 'yes' ]; then
         exit $RESULT

From 36404a52ab09770e775fbc43132c43d6c09d0588 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 18 Nov 2022 12:58:34 +0100
Subject: [PATCH 198/232] ci: add DIAGNOSTIC=1 to inspect rebar3 error

---
 .github/workflows/run_test_cases.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index d1794f2fb..2233dd1c8 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -39,6 +39,7 @@ jobs:
         - name: get_all_deps
           working-directory: source
           run: |
+            export DIAGNOSTIC=1
             make deps-all
             ./rebar3 as test compile
             cd ..

From 3d75a1db3b04e09f7f15bb3b73754f215454cdd8 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 18 Nov 2022 13:15:44 +0100
Subject: [PATCH 199/232] ci: refactor run_test_cases to prepare ce and ee
 source zips

---
 .github/workflows/run_test_cases.yaml | 99 +++++++++++++++++----------
 1 file changed, 62 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 2233dd1c8..58e95ea2d 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -14,7 +14,7 @@ on:
   pull_request:
 
 jobs:
-    prepare:
+    prepare_ce:
         runs-on: aws-amd64
         # prepare source with any OTP version, no need for a matrix
         container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
@@ -25,7 +25,6 @@ jobs:
         - uses: actions/checkout@v3
           with:
             path: source
-            fetch-depth: 0
         - name: find_ct_apps
           working-directory: source
           id: run_find_apps
@@ -34,23 +33,65 @@ jobs:
             docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
             echo "fast-ct-apps: $fast_ct_apps"
             echo "docer-ct-apps: $docker_ct_apps"
-            echo "::set-output name=fast_ct_apps::$fast_ct_apps"
-            echo "::set-output name=docker_ct_apps::$docker_ct_apps"
+            echo "::set-output name=fast_ct_apps_ce::$fast_ct_apps"
+            echo "::set-output name=docker_ct_apps_ce::$docker_ct_apps"
         - name: get_all_deps
           working-directory: source
+          env:
+            PROFILE: emqx
+            DIAGNOSTIC: 1
           run: |
-            export DIAGNOSTIC=1
-            make deps-all
+            make ensure-rebar3
+            # this will fetch all deps and compile
             ./rebar3 as test compile
             cd ..
             zip -ryq source.zip source/* source/.[^.]*
         - uses: actions/upload-artifact@v3
           with:
-            name: source
+            name: source-emqx
+            path: source.zip
+
+    prepare_ee:
+        runs-on: aws-amd64
+        # prepare source with any OTP version, no need for a matrix
+        container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
+        outputs:
+          fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
+          docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
+        steps:
+        - uses: actions/checkout@v3
+          with:
+            path: source
+        - name: find_ct_apps
+          working-directory: source
+          id: run_find_apps
+          run: |
+            fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)"
+            docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
+            echo "fast-ct-apps: $fast_ct_apps"
+            echo "docer-ct-apps: $docker_ct_apps"
+            echo "::set-output name=fast_ct_apps_ee::$fast_ct_apps"
+            echo "::set-output name=docker_ct_apps_ee::$docker_ct_apps"
+        - name: get_all_deps
+          working-directory: source
+          env:
+            PROFILE: emqx-enterprise
+            DIAGNOSTIC: 1
+          run: |
+            make ensure-rebar3
+            # this will fetch all deps and compile
+            ./rebar3 as test compile
+            cd ..
+            zip -ryq source.zip source/* source/.[^.]*
+        - uses: actions/upload-artifact@v3
+          with:
+            name: source-emqx-enterprise
             path: source.zip
 
     eunit_and_proper:
-        needs: prepare
+        needs:
+          - prepare_ee
+          - prepare_ce
         runs-on: aws-amd64
         strategy:
           fail-fast: false
@@ -67,7 +108,7 @@ jobs:
         - uses: AutoModality/action-clean@v1
         - uses: actions/download-artifact@v3
           with:
-            name: source
+            name: source-${{ matrix.profile }}
             path: .
         - name: unzip source code
           env:
@@ -93,10 +134,15 @@ jobs:
             path: source/_build/test/cover
 
     ct_docker:
-        needs: prepare
+        needs:
+          - prepare_ee
+          - prepare_ce
         strategy:
           fail-fast: false
           matrix:
+            profile:
+              - emqx
+              - emqx-enterprise
             app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }}
             otp_release:
               - "erlang24"
@@ -110,7 +156,7 @@ jobs:
         - uses: AutoModality/action-clean@v1
         - uses: actions/download-artifact@v3
           with:
-            name: source
+            name: source-${{ matrix.profile }}
             path: .
         - name: unzip source code
           run: unzip -q source.zip
@@ -135,7 +181,9 @@ jobs:
             path: source/_build/test/logs
 
     ct:
-        needs: prepare
+        needs:
+          - prepare_ee
+          - prepare_ce
         strategy:
           fail-fast: false
           matrix:
@@ -154,7 +202,7 @@ jobs:
         - uses: AutoModality/action-clean@v1
         - uses: actions/download-artifact@v3
           with:
-            name: source
+            name: source-${{ matrix.profile }}
             path: .
         - name: unzip source code
           run: unzip -q source.zip
@@ -166,16 +214,7 @@ jobs:
             PROFILE: ${{ matrix.profile }}
             WHICH_APP: ${{ matrix.app_name }}
           run: |
-            if [ "$PROFILE" = 'emqx-enterprise' ]; then
-              COMPILE_FLAGS="$(grep -R "EMQX_RELEASE_EDITION" "$WHICH_APP" | wc -l || true)"
-              if [ "$COMPILE_FLAGS" -gt 0 ]; then
-                # need to clean first because the default profile was
-                make clean
-                make "${WHICH_APP}-ct"
-              else
-                echo "skip_common_test_run_for_app ${WHICH_APP}-ct"
-              fi
-            else
+            if [[ "$PROFILE" != 'emqx-enterprise' ]]; then
               case "$WHICH_APP" in
                 lib-ee/*)
                   echo "skip_opensource_edition_test_for_lib-ee"
@@ -245,17 +284,3 @@ jobs:
             curl -v -k https://coveralls.io/webhook \
                  --header "Content-Type: application/json" \
                  --data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true
-
-    allgood_functional_tests:
-      runs-on: ubuntu-20.04
-      needs:
-        - eunit_and_proper
-        - ct_docker
-        - ct
-      steps:
-      - name: Check if all functional tests succeeded
-        uses: re-actors/alls-green@release/v1
-        with:
-          #allowed-failures:
-          #allowed-skips:
-          jobs: ${{ toJSON(needs) }}

From 06cb7aeb486e659a6aca35b08eaa7491e15ca209 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 18 Nov 2022 19:06:51 +0100
Subject: [PATCH 200/232] ci: make ci matrix from find-apps.sh

---
 scripts/find-apps.sh | 89 ++++++++++++++++++++++++++++----------------
 1 file changed, 56 insertions(+), 33 deletions(-)

diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh
index 8c41fecbf..65585f9fd 100755
--- a/scripts/find-apps.sh
+++ b/scripts/find-apps.sh
@@ -8,25 +8,18 @@ cd -P -- "$(dirname -- "$0")/.."
 help() {
     echo
     echo "-h|--help:        To display this usage info"
-    echo "--ct fast|docker: Print apps which needs docker-compose to run ct"
-    echo "--json:           Print apps in json"
+    echo "--ci fast|docker: Print apps in json format for github ci mtrix"
 }
 
-WANT_JSON='no'
-CT='novalue'
+CI='fast'
 while [ "$#" -gt 0 ]; do
     case $1 in
         -h|--help)
             help
             exit 0
             ;;
-        --json)
-            WANT_JSON='yes'
-            shift 1
-            ;;
-
-        --ct)
-            CT="$2"
+        --ci)
+            CI="$2"
             shift 2
             ;;
         *)
@@ -52,27 +45,57 @@ CE="$(find_app 'apps')"
 EE="$(find_app 'lib-ee')"
 APPS_ALL="$(echo -e "${CE}\n${EE}")"
 
-if [ "$CT" = 'novalue' ]; then
-    RESULT="${APPS_ALL}"
-else
-    APPS_NORMAL_CT=( )
-    APPS_DOCKER_CT=( )
-    for app in ${APPS_ALL}; do
-        if [ -f "${app}/docker-ct" ]; then
-            APPS_DOCKER_CT+=("$app")
-        else
-            APPS_NORMAL_CT+=("$app")
-        fi
-    done
-    if [ "$CT" = 'docker' ]; then
-        RESULT="${APPS_DOCKER_CT[*]}"
-    else
-        RESULT="${APPS_NORMAL_CT[*]}"
-    fi
+if [ "$CI" = 'no' ]; then
+    echo "${APPS_ALL}"
+    exit 0
 fi
 
-if [ "$WANT_JSON" = 'yes' ]; then
-    echo "${RESULT}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
-else
-    echo "${RESULT}" | xargs
-fi
+##################################################
+###### now deal with the github action's matrix.
+##################################################
+
+dimensions() {
+    app="$1"
+    if [ -f "${app}/docker-ct" ]; then
+        if [[ "$CI" != 'docker' ]]; then
+            return
+        fi
+    else
+        if [[ "$CI" != 'fast' ]]; then
+            return
+        fi
+    fi
+    case "${app}" in
+        apps/*)
+            profile='emqx'
+            ;;
+        lib-ee/*)
+            profile='emqx-enterprise'
+            ;;
+        *)
+            echo "unknown app: $app"
+            exit 1
+            ;;
+    esac
+    echo -n "$app $profile" | jq -R -s -c 'split(" ")'
+}
+
+matrix() {
+    first_row='yes'
+    for app in ${APPS_ALL}; do
+        row="$(dimensions "$app")"
+        if [ -z "$row" ]; then
+            continue
+        fi
+        if [ "$first_row" = 'yes' ]; then
+            first_row='no'
+            echo -n "$row"
+        else
+            echo -n ",${row}"
+        fi
+    done
+}
+
+echo -n '['
+matrix
+echo ']'

From aeac9bf43f794e6dc4a622cf55a25049ad4f3425 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 18 Nov 2022 19:22:02 +0100
Subject: [PATCH 201/232] ci: refactor, find-app.sh produced matrix

---
 .github/workflows/run_test_cases.yaml | 82 +++++++++------------------
 1 file changed, 28 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 58e95ea2d..7a9adb24e 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -14,27 +14,27 @@ on:
   pull_request:
 
 jobs:
-    prepare_ce:
+    prepare:
         runs-on: aws-amd64
         # prepare source with any OTP version, no need for a matrix
         container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
         outputs:
-          fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
-          docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
+          fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }}
+          docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }}
         steps:
         - uses: actions/checkout@v3
           with:
             path: source
-        - name: find_ct_apps
+        - name: Find CT Apps
           working-directory: source
-          id: run_find_apps
+          id: find_ct_apps
           run: |
-            fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)"
-            docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
-            echo "fast-ct-apps: $fast_ct_apps"
-            echo "docer-ct-apps: $docker_ct_apps"
-            echo "::set-output name=fast_ct_apps_ce::$fast_ct_apps"
-            echo "::set-output name=docker_ct_apps_ce::$docker_ct_apps"
+            fast_ct_apps="$(./scripts/find-apps.sh --ci fast)"
+            docker_ct_apps="$(./scripts/find-apps.sh --ci docker)"
+            echo "fast: $fast_ct_apps"
+            echo "docker: $docker_ct_apps"
+            echo "::set-output name=fast_ct_apps::$fast_ct_apps"
+            echo "::set-output name=docker_ct_apps::$docker_ct_apps"
         - name: get_all_deps
           working-directory: source
           env:
@@ -55,23 +55,10 @@ jobs:
         runs-on: aws-amd64
         # prepare source with any OTP version, no need for a matrix
         container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
-        outputs:
-          fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
-          docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
         steps:
         - uses: actions/checkout@v3
           with:
             path: source
-        - name: find_ct_apps
-          working-directory: source
-          id: run_find_apps
-          run: |
-            fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)"
-            docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
-            echo "fast-ct-apps: $fast_ct_apps"
-            echo "docer-ct-apps: $docker_ct_apps"
-            echo "::set-output name=fast_ct_apps_ee::$fast_ct_apps"
-            echo "::set-output name=docker_ct_apps_ee::$docker_ct_apps"
         - name: get_all_deps
           working-directory: source
           env:
@@ -90,8 +77,8 @@ jobs:
 
     eunit_and_proper:
         needs:
+          - prepare
           - prepare_ee
-          - prepare_ce
         runs-on: aws-amd64
         strategy:
           fail-fast: false
@@ -135,15 +122,12 @@ jobs:
 
     ct_docker:
         needs:
+          - prepare
           - prepare_ee
-          - prepare_ce
         strategy:
           fail-fast: false
           matrix:
-            profile:
-              - emqx
-              - emqx-enterprise
-            app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }}
+            app: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }}
             otp_release:
               - "erlang24"
 
@@ -156,7 +140,7 @@ jobs:
         - uses: AutoModality/action-clean@v1
         - uses: actions/download-artifact@v3
           with:
-            name: source-${{ matrix.profile }}
+            name: source-${{ matrix.app[1] }}
             path: .
         - name: unzip source code
           run: unzip -q source.zip
@@ -167,9 +151,12 @@ jobs:
             MYSQL_TAG: 8
             PGSQL_TAG: 13
             REDIS_TAG: 6
+            WHICH_APP: ${{ matrix.app[0] }}
+            PROFILE: ${{ matrix.app[1] }}
           run: |
+            echo $PROFILE
             rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt
-            ./scripts/ct/run.sh --app ${{ matrix.app_name }}
+            ./scripts/ct/run.sh --app $WHICH_APP
         - uses: actions/upload-artifact@v3
           with:
             name: coverdata
@@ -177,21 +164,17 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs_${{ matrix.otp_release }}-${{ matrix.profile }}
+            name: logs_${{ matrix.otp_release }}-${{ matrix.app[0] }}-${{ matrix.app[1] }}
             path: source/_build/test/logs
 
     ct:
         needs:
+          - prepare
           - prepare_ee
-          - prepare_ce
         strategy:
           fail-fast: false
           matrix:
-            app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
-            profile:
-              - emqx
-              - emqx-enterprise
-
+            app: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
         runs-on: aws-amd64
         container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
         defaults:
@@ -202,28 +185,19 @@ jobs:
         - uses: AutoModality/action-clean@v1
         - uses: actions/download-artifact@v3
           with:
-            name: source-${{ matrix.profile }}
+            name: source-${{ matrix.app[1] }}
             path: .
         - name: unzip source code
           run: unzip -q source.zip
 
-          # produces <app-name>.coverdata
+          # produces $PROFILE-<app-name>.coverdata
         - name: run common test
           working-directory: source
           env:
-            PROFILE: ${{ matrix.profile }}
-            WHICH_APP: ${{ matrix.app_name }}
+            WHICH_APP: ${{ matrix.app[0] }}
+            PROFILE: ${{ matrix.app[1] }}
           run: |
-            if [[ "$PROFILE" != 'emqx-enterprise' ]]; then
-              case "$WHICH_APP" in
-                lib-ee/*)
-                  echo "skip_opensource_edition_test_for_lib-ee"
-                  ;;
-                *)
-                  make "${WHICH_APP}-ct"
-                  ;;
-              esac
-            fi
+            make "${WHICH_APP}-ct"
         - uses: actions/upload-artifact@v3
           with:
             name: coverdata
@@ -232,7 +206,7 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs_${{ matrix.otp_release }}-${{ matrix.profile }}
+            name: logs_${{ matrix.otp_release }}-${{ matrix.app[0] }}-${{ matrix.app[1] }}
             path: source/_build/test/logs
 
     make_cover:

From 90ad3c70bf9db1b8d0b5c88231f58cb601121bea Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 18 Nov 2022 19:58:15 +0100
Subject: [PATCH 202/232] chore: upgrade to replayq 0.3.5

---
 mix.exs      | 2 +-
 rebar.config | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index 39a12769a..c7d9ae856 100644
--- a/mix.exs
+++ b/mix.exs
@@ -57,7 +57,7 @@ defmodule EMQXUmbrella.MixProject do
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.7", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true},
-      {:replayq, github: "emqx/replayq", tag: "0.3.4", override: true},
+      {:replayq, github: "emqx/replayq", tag: "0.3.5", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
       {:emqtt, github: "emqx/emqtt", tag: "1.7.0-rc.2", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
diff --git a/rebar.config b/rebar.config
index c2e62d3aa..369da655a 100644
--- a/rebar.config
+++ b/rebar.config
@@ -59,7 +59,7 @@
     , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.7"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
-    , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.4"}}}
+    , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.5"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0-rc.2"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}

From e4a4956c19a93c8052b925eefd96551085b5a7a9 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 18 Nov 2022 19:27:09 +0100
Subject: [PATCH 203/232] ci: fix find-apps.sh default printout

---
 .github/workflows/codeball.yml        | 17 -----------------
 .github/workflows/run_test_cases.yaml |  9 ++++++---
 scripts/find-apps.sh                  |  7 ++++---
 3 files changed, 10 insertions(+), 23 deletions(-)
 delete mode 100644 .github/workflows/codeball.yml

diff --git a/.github/workflows/codeball.yml b/.github/workflows/codeball.yml
deleted file mode 100644
index ed69d5c10..000000000
--- a/.github/workflows/codeball.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-name: Codeball
-on: [pull_request]
-
-jobs:
-  codeball_job:
-    runs-on: ubuntu-latest
-    name: Codeball
-    steps:
-      # Run Codeball on all new Pull Requests 🚀
-      # For customizations and more documentation, see https://github.com/sturdy-dev/codeball-action
-      - name: Codeball
-        uses: sturdy-dev/codeball-action@v2
-        with:
-          approvePullRequests: "true"
-          labelPullRequestsWhenApproved: "true"
-          labelPullRequestsWhenReviewNeeded: "false"
-          failJobsWhenReviewNeeded: "false"
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 9e2e12b9a..07ae00a50 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -39,7 +39,7 @@ jobs:
           working-directory: source
           env:
             PROFILE: emqx
-            DIAGNOSTIC: 1
+              #DIAGNOSTIC: 1
           run: |
             make ensure-rebar3
             # this will fetch all deps and compile
@@ -63,7 +63,7 @@ jobs:
           working-directory: source
           env:
             PROFILE: emqx-enterprise
-            DIAGNOSTIC: 1
+              #DIAGNOSTIC: 1
           run: |
             make ensure-rebar3
             # this will fetch all deps and compile
@@ -221,7 +221,7 @@ jobs:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source
+          name: source-emqx-enterprise
           path: .
       - name: unzip source code
         run: unzip -q source.zip
@@ -234,12 +234,15 @@ jobs:
 
       - name: make cover
         working-directory: source
+        env:
+          PROFILE: emqx-enterprise
         run: make cover
 
       - name: send to coveralls
         working-directory: source
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PROFILE: emqx-enterprise
         run: make coveralls
 
       - name: get coveralls logs
diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh
index 65585f9fd..fefb73568 100755
--- a/scripts/find-apps.sh
+++ b/scripts/find-apps.sh
@@ -11,7 +11,7 @@ help() {
     echo "--ci fast|docker: Print apps in json format for github ci mtrix"
 }
 
-CI='fast'
+CI='novalue'
 while [ "$#" -gt 0 ]; do
     case $1 in
         -h|--help)
@@ -45,7 +45,7 @@ CE="$(find_app 'apps')"
 EE="$(find_app 'lib-ee')"
 APPS_ALL="$(echo -e "${CE}\n${EE}")"
 
-if [ "$CI" = 'no' ]; then
+if [ "$CI" = 'novalue' ]; then
     echo "${APPS_ALL}"
     exit 0
 fi
@@ -77,7 +77,8 @@ dimensions() {
             exit 1
             ;;
     esac
-    echo -n "$app $profile" | jq -R -s -c 'split(" ")'
+    ## poor-man's json formatter
+    echo -n -e "[\"$app\", \"$profile\"]"
 }
 
 matrix() {

From c878c20499c340b16696d900cc9db0b4a6d68331 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 20 Nov 2022 08:49:05 +0100
Subject: [PATCH 204/232] ci: exclude enterprise tag for windows builds

---
 .github/workflows/build_packages.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index b738ab999..9f27d5012 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -86,6 +86,7 @@ jobs:
 
   windows:
     runs-on: windows-2019
+    if: ! startsWith(github.ref_name, 'e')
     needs: prepare
     strategy:
       fail-fast: false

From 03d3d1d65a17f00bb21468ba7fd2c879f0220721 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 20 Nov 2022 08:50:11 +0100
Subject: [PATCH 205/232] ci: exclude raspbian all together

the 'exclude' matrix prior to this change had raspbian all
excluded anyway.
---
 .github/workflows/build_packages.yaml | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 9f27d5012..57e8f58ad 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -208,7 +208,6 @@ jobs:
           - debian9
           - el8
           - el7
-          - raspbian10
         build_machine:
           - aws-arm64
           - ubuntu-20.04
@@ -217,20 +216,6 @@ jobs:
           build_machine: ubuntu-20.04
         - arch: amd64
           build_machine: aws-arm64
-        - os: raspbian9
-          arch: amd64
-        - os: raspbian10
-          arch: amd64
-        - os: raspbian10 # we only have arm32 image
-          arch: arm64
-        - os: raspbian9
-          profile: emqx
-        - os: raspbian10
-          profile: emqx
-        - os: raspbian9
-          profile: emqx-enterprise
-        - os: raspbian10
-          profile: emqx-enterprise
         include:
           - profile: emqx
             otp: 24.3.4.2-1

From c3777e4920f175b40ba230a97e451fdae12f4726 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 20 Nov 2022 09:06:54 +0100
Subject: [PATCH 206/232] ci: avoid using 'include' for elixir builds

prior to this change the elixir builds matrix dimensions were 'included'
cusing it to build for enterprise eidtion

use exclude instead.

also removed otp from the artifact upload name, leave only profile,
so there is no need to 'include' with a '-windows' suffix for
windows package uploads
---
 .github/workflows/build_packages.yaml | 56 +++++++++++----------------
 1 file changed, 22 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 57e8f58ad..eb9ba5f57 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -93,8 +93,6 @@ jobs:
       matrix:
         profile: # for now only CE for windows
           - emqx
-        otp:
-          - 24.2.1
     steps:
     - uses: actions/download-artifact@v3
       with:
@@ -105,7 +103,7 @@ jobs:
     - uses: ilammy/msvc-dev-cmd@v1.12.0
     - uses: erlef/setup-beam@v1
       with:
-        otp-version: ${{ matrix.otp }}
+        otp-version: 24.2.1
     - name: build
       env:
         PYTHON: python
@@ -130,7 +128,7 @@ jobs:
         echo "EMQX uninstalled"
     - uses: actions/upload-artifact@v3
       with:
-        name: ${{ matrix.profile }}-windows
+        name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/
 
   mac:
@@ -169,7 +167,7 @@ jobs:
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
     - uses: actions/upload-artifact@v3
       with:
-        name: ${{ matrix.profile }}-${{ matrix.otp }}
+        name: ${{ matrix.profile }}
         path: _packages/${{ matrix.profile }}/
 
   linux:
@@ -184,7 +182,7 @@ jobs:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
         otp:
-          - 24.3.4.2-1 # we test with OTP 23, but only build package on OTP 24 versions
+          - 24.3.4.2-1
         elixir:
           - 1.13.4
         # used to split elixir packages into a separate job, since the
@@ -202,36 +200,31 @@ jobs:
         os:
           - ubuntu20.04
           - ubuntu18.04
-          - ubuntu16.04
           - debian11
           - debian10
-          - debian9
           - el8
           - el7
         build_machine:
           - aws-arm64
           - ubuntu-20.04
         exclude:
-        - arch: arm64
-          build_machine: ubuntu-20.04
-        - arch: amd64
-          build_machine: aws-arm64
-        include:
-          - profile: emqx
-            otp: 24.3.4.2-1
-            elixir: 1.13.4
-            build_elixir: with_elixir
-            arch: amd64
-            os: ubuntu20.04
+          - arch: arm64
             build_machine: ubuntu-20.04
-          - profile: emqx
-            otp: 24.3.4.2-1
-            elixir: 1.13.4
-            build_elixir: with_elixir
-            arch: amd64
-            os: el8
-            build_machine: ubuntu-20.04
-
+          - arch: amd64
+            build_machine: aws-arm64
+            # elixir: only for opensource edition and only on ubuntu20.04 and el8 on amd64
+          - build_elixir: with_elixir
+            profile: emqx-enterprise
+          - build_elixir: with_elixir
+            arch: arm64
+          - build_elixir: with_elixir
+            os: ubuntu18.04
+          - build_elixir: with_elixir
+            os: debian10
+          - build_elixir: with_elixir
+            os: debian11
+          - build_elixir: with_elixir
+            os: el7
     defaults:
       run:
         shell: bash
@@ -280,7 +273,7 @@ jobs:
         done
     - uses: actions/upload-artifact@v3
       with:
-        name: ${{ matrix.profile }}-${{ matrix.otp }}
+        name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/
 
   publish_artifacts:
@@ -292,15 +285,10 @@ jobs:
       matrix:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
-        otp:
-          - 24.3.4.2-1
-        include:
-          - profile: emqx
-            otp: windows # otp version on windows is rather fixed
     steps:
     - uses: actions/download-artifact@v3
       with:
-        name: ${{ matrix.profile }}-${{ matrix.otp }}
+        name: ${{ matrix.profile }}
         path: packages/${{ matrix.profile }}
     - name: install dos2unix
       run: sudo apt-get update && sudo apt install -y dos2unix

From 458d23ebc4281c0811f9806e9ec6cfe65b1f50dd Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 20 Nov 2022 09:11:53 +0100
Subject: [PATCH 207/232] chore: bump version to v5.0.0-beta.5

---
 apps/emqx/include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index 898b9551f..2b5d8727e 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -35,7 +35,7 @@
 -define(EMQX_RELEASE_CE, "5.0.10").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.0.0-beta.4").
+-define(EMQX_RELEASE_EE, "5.0.0-beta.5").
 
 %% the HTTP API version
 -define(EMQX_API_VERSION, "5.0").

From c559334e69c9dd23e199af5b035f640b476eccdd Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 20 Nov 2022 09:13:58 +0100
Subject: [PATCH 208/232] ci: fix windows build

---
 .github/workflows/build_packages.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index eb9ba5f57..6a2816cbe 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -86,7 +86,7 @@ jobs:
 
   windows:
     runs-on: windows-2019
-    if: ! startsWith(github.ref_name, 'e')
+    if: startsWith(github.ref_name, 'v')
     needs: prepare
     strategy:
       fail-fast: false

From b3986176149e2834974aab9f54ba0468497d86c3 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 28 Nov 2022 21:12:43 +0100
Subject: [PATCH 209/232] chore: bump app versions

---
 apps/emqx/src/emqx.app.src                         | 2 +-
 apps/emqx_authn/src/emqx_authn.app.src             | 2 +-
 apps/emqx_authz/src/emqx_authz.app.src             | 2 +-
 apps/emqx_bridge/src/emqx_bridge.app.src           | 2 +-
 apps/emqx_connector/src/emqx_connector.app.src     | 2 +-
 apps/emqx_dashboard/src/emqx_dashboard.app.src     | 2 +-
 apps/emqx_management/src/emqx_management.app.src   | 2 +-
 apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src
index 3d1fe32d3..e687e9905 100644
--- a/apps/emqx/src/emqx.app.src
+++ b/apps/emqx/src/emqx.app.src
@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.11"},
+    {vsn, "5.0.12"},
     {modules, []},
     {registered, []},
     {applications, [
diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index 038d2601a..7f0305871 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.9"},
+    {vsn, "0.1.10"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},
diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index e6913b43c..3f0f96e72 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.8"},
+    {vsn, "0.1.9"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src
index fcaf9394f..3cc858665 100644
--- a/apps/emqx_bridge/src/emqx_bridge.app.src
+++ b/apps/emqx_bridge/src/emqx_bridge.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "EMQX bridges"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [
diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src
index 547a37b8e..1bfce0735 100644
--- a/apps/emqx_connector/src/emqx_connector.app.src
+++ b/apps/emqx_connector/src/emqx_connector.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "An OTP application"},
-    {vsn, "0.1.8"},
+    {vsn, "0.1.9"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index 9e639bcf8..16c51342f 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.8"},
+    {vsn, "5.0.9"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},
diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index ab726cbb2..a74d411f9 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.8"},
+    {vsn, "5.0.9"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 6419e4184..8608cd67a 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.4"},
+    {vsn, "5.0.5"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt]},

From 86e86c5a85d7fc98f225565835f3f14de8fa9409 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 09:45:56 +0100
Subject: [PATCH 210/232] ci: clean before prepare and 'make' the default
 target

the eunit & proper step depends on a pre-built default target
---
 .github/workflows/run_test_cases.yaml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 07ae00a50..cb4e1e409 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -22,6 +22,7 @@ jobs:
           fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }}
           docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }}
         steps:
+        - uses: AutoModality/action-clean@v1
         - uses: actions/checkout@v3
           with:
             path: source
@@ -42,8 +43,8 @@ jobs:
               #DIAGNOSTIC: 1
           run: |
             make ensure-rebar3
-            # this will fetch all deps and compile
-            ./rebar3 as test compile
+            # fetch all deps and compile
+            make
             cd ..
             zip -ryq source.zip source/* source/.[^.]*
         - uses: actions/upload-artifact@v3
@@ -56,6 +57,7 @@ jobs:
         # prepare source with any OTP version, no need for a matrix
         container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
         steps:
+        - uses: AutoModality/action-clean@v1
         - uses: actions/checkout@v3
           with:
             path: source
@@ -66,8 +68,8 @@ jobs:
               #DIAGNOSTIC: 1
           run: |
             make ensure-rebar3
-            # this will fetch all deps and compile
-            ./rebar3 as test compile
+            # fetch all deps and compile
+            make
             cd ..
             zip -ryq source.zip source/* source/.[^.]*
         - uses: actions/upload-artifact@v3

From 9382399728b58adc194df3d2af0221d764915c9c Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 12:52:26 +0100
Subject: [PATCH 211/232] chore: add an empty file emqx_license.conf to make ci
 happy

---
 lib-ee/emqx_license/etc/emqx_license.conf | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 lib-ee/emqx_license/etc/emqx_license.conf

diff --git a/lib-ee/emqx_license/etc/emqx_license.conf b/lib-ee/emqx_license/etc/emqx_license.conf
new file mode 100644
index 000000000..e69de29bb

From c53a81555521e4b4484805c739d7dc875b43ea65 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 12:52:51 +0100
Subject: [PATCH 212/232] ci: pre-compile test profile

---
 .github/workflows/run_test_cases.yaml | 6 ++++--
 Makefile                              | 4 ++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index cb4e1e409..7c6b039fc 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -44,7 +44,8 @@ jobs:
           run: |
             make ensure-rebar3
             # fetch all deps and compile
-            make
+            make emqx
+            make test-compile
             cd ..
             zip -ryq source.zip source/* source/.[^.]*
         - uses: actions/upload-artifact@v3
@@ -69,7 +70,8 @@ jobs:
           run: |
             make ensure-rebar3
             # fetch all deps and compile
-            make
+            make emqx-enterprise
+            make test-compile
             cd ..
             zip -ryq source.zip source/* source/.[^.]*
         - uses: actions/upload-artifact@v3
diff --git a/Makefile b/Makefile
index 311aed651..8cec47020 100644
--- a/Makefile
+++ b/Makefile
@@ -68,6 +68,10 @@ eunit: $(REBAR) conf-segs
 proper: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) proper -d test/props -c
 
+.PHONY: test-compile
+test-compile: $(REBAR) conf-segs
+	$(REBAR) as test compile
+
 .PHONY: ct
 ct: $(REBAR) conf-segs
 	@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-ct

From cc9e5b1a562e79276ca31ca4aacd5ea6ad8c9830 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 14:01:26 +0100
Subject: [PATCH 213/232] ci(scripts/ct/run.sh): fix undefined vars when runing
 ct for non-docker

---
 scripts/ct/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index edfaf6a9e..5a2cce036 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -83,7 +83,7 @@ if [ -f "$DOCKER_CT_ENVS_FILE" ]; then
     # shellcheck disable=SC2002
     CT_DEPS="$(cat "$DOCKER_CT_ENVS_FILE" | xargs)"
 fi
-CT_DEPS="${ERLANG_CONTAINER} ${CT_DEPS}"
+CT_DEPS="${ERLANG_CONTAINER} ${CT_DEPS:-}"
 
 FILES=( )
 

From 983e9048588584c302eb789c0582321a026edb0a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 14:02:27 +0100
Subject: [PATCH 214/232] fix(connector): fix ssl clear

---
 apps/emqx/src/emqx_map_lib.erl                |  2 +-
 apps/emqx_bridge/test/emqx_bridge_SUITE.erl   | 98 +++++--------------
 .../emqx_connector/src/emqx_connector_ssl.erl | 42 +++-----
 3 files changed, 38 insertions(+), 104 deletions(-)

diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl
index b01391c7b..6484d4269 100644
--- a/apps/emqx/src/emqx_map_lib.erl
+++ b/apps/emqx/src/emqx_map_lib.erl
@@ -133,7 +133,7 @@ deep_merge(BaseMap, NewMap) ->
     ),
     maps:merge(MergedBase, maps:with(NewKeys, NewMap)).
 
--spec deep_convert(map(), convert_fun(), Args :: list()) -> map().
+-spec deep_convert(any(), convert_fun(), Args :: list()) -> any().
 deep_convert(Map, ConvFun, Args) when is_map(Map) ->
     maps:fold(
         fun(K, V, Acc) ->
diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
index 5b19904ff..99d5af447 100644
--- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
@@ -44,6 +44,9 @@ init_per_testcase(t_get_basic_usage_info_1, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     setup_fake_telemetry_data(),
     Config;
+init_per_testcase(t_update_ssl_conf, Config) ->
+    Path = [bridges, <<"mqtt">>, <<"ssl_update_test">>],
+    [{config_path, Path} | Config];
 init_per_testcase(_TestCase, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     Config.
@@ -63,6 +66,9 @@ end_per_testcase(t_get_basic_usage_info_1, _Config) ->
     ok = emqx_config:put([bridges], #{}),
     ok = emqx_config:put_raw([bridges], #{}),
     ok;
+end_per_testcase(t_update_ssl_conf, Config) ->
+    Path = proplists:get_value(config_path, Config),
+    emqx:remove_config(Path);
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
@@ -149,82 +155,30 @@ setup_fake_telemetry_data() ->
     ok = snabbkaffe:stop(),
     ok.
 
-t_update_ssl_conf(_) ->
-    Path = [bridges, <<"mqtt">>, <<"ssl_update_test">>],
+t_update_ssl_conf(Config) ->
+    Path = proplists:get_value(config_path, Config),
     EnableSSLConf = #{
-        <<"connector">> =>
+        <<"bridge_mode">> => false,
+        <<"clean_start">> => true,
+        <<"keepalive">> => <<"60s">>,
+        <<"mode">> => <<"cluster_shareload">>,
+        <<"proto_ver">> => <<"v4">>,
+        <<"server">> => <<"127.0.0.1:1883">>,
+        <<"ssl">> =>
             #{
-                <<"bridge_mode">> => false,
-                <<"clean_start">> => true,
-                <<"keepalive">> => <<"60s">>,
-                <<"mode">> => <<"cluster_shareload">>,
-                <<"proto_ver">> => <<"v4">>,
-                <<"server">> => <<"127.0.0.1:1883">>,
-                <<"ssl">> =>
-                    #{
-                        <<"cacertfile">> => cert_file("cafile"),
-                        <<"certfile">> => cert_file("certfile"),
-                        <<"enable">> => true,
-                        <<"keyfile">> => cert_file("keyfile"),
-                        <<"verify">> => <<"verify_peer">>
-                    }
-            },
-        <<"direction">> => <<"ingress">>,
-        <<"local_qos">> => 1,
-        <<"payload">> => <<"${payload}">>,
-        <<"remote_qos">> => 1,
-        <<"remote_topic">> => <<"t/#">>,
-        <<"retain">> => false
+                <<"cacertfile">> => cert_file("cafile"),
+                <<"certfile">> => cert_file("certfile"),
+                <<"enable">> => true,
+                <<"keyfile">> => cert_file("keyfile"),
+                <<"verify">> => <<"verify_peer">>
+            }
     },
-
-    emqx:update_config(Path, EnableSSLConf),
-    ?assertMatch({ok, [_, _, _]}, list_pem_dir(Path)),
-    NoSSLConf = #{
-        <<"connector">> =>
-            #{
-                <<"bridge_mode">> => false,
-                <<"clean_start">> => true,
-                <<"keepalive">> => <<"60s">>,
-                <<"max_inflight">> => 32,
-                <<"mode">> => <<"cluster_shareload">>,
-                <<"password">> => <<>>,
-                <<"proto_ver">> => <<"v4">>,
-                <<"reconnect_interval">> => <<"15s">>,
-                <<"replayq">> =>
-                    #{<<"offload">> => false, <<"seg_bytes">> => <<"100MB">>},
-                <<"retry_interval">> => <<"15s">>,
-                <<"server">> => <<"127.0.0.1:1883">>,
-                <<"ssl">> =>
-                    #{
-                        <<"ciphers">> => <<>>,
-                        <<"depth">> => 10,
-                        <<"enable">> => false,
-                        <<"reuse_sessions">> => true,
-                        <<"secure_renegotiate">> => true,
-                        <<"user_lookup_fun">> => <<"emqx_tls_psk:lookup">>,
-                        <<"verify">> => <<"verify_peer">>,
-                        <<"versions">> =>
-                            [
-                                <<"tlsv1.3">>,
-                                <<"tlsv1.2">>,
-                                <<"tlsv1.1">>,
-                                <<"tlsv1">>
-                            ]
-                    },
-                <<"username">> => <<>>
-            },
-        <<"direction">> => <<"ingress">>,
-        <<"enable">> => true,
-        <<"local_qos">> => 1,
-        <<"payload">> => <<"${payload}">>,
-        <<"remote_qos">> => 1,
-        <<"remote_topic">> => <<"t/#">>,
-        <<"retain">> => false
-    },
-
-    emqx:update_config(Path, NoSSLConf),
+    {ok, _} = emqx:update_config(Path, EnableSSLConf),
+    {ok, Certs} = list_pem_dir(Path),
+    ?assertMatch([_, _, _], Certs),
+    NoSSLConf = EnableSSLConf#{<<"ssl">> := #{<<"enable">> => false}},
+    {ok, _} = emqx:update_config(Path, NoSSLConf),
     ?assertMatch({error, not_dir}, list_pem_dir(Path)),
-    emqx:remove_config(Path),
     ok.
 
 list_pem_dir(Path) ->
diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl
index 7dc6179e1..c2449f095 100644
--- a/apps/emqx_connector/src/emqx_connector_ssl.erl
+++ b/apps/emqx_connector/src/emqx_connector_ssl.erl
@@ -24,20 +24,6 @@
     try_clear_certs/3
 ]).
 
-%% TODO: rm `connector` case after `dev/ee5.0` merged into `master`.
-%% The `connector` config layer will be removed.
-%% for bridges with `connector` field. i.e. `mqtt_source` and `mqtt_sink`
-convert_certs(RltvDir, #{<<"connector">> := Connector} = Config) when
-    is_map(Connector)
-->
-    SSL = maps:get(<<"ssl">>, Connector, undefined),
-    new_ssl_config(RltvDir, Config, SSL);
-convert_certs(RltvDir, #{connector := Connector} = Config) when
-    is_map(Connector)
-->
-    SSL = maps:get(ssl, Connector, undefined),
-    new_ssl_config(RltvDir, Config, SSL);
-%% for bridges without `connector` field. i.e. webhook
 convert_certs(RltvDir, #{<<"ssl">> := SSL} = Config) ->
     new_ssl_config(RltvDir, Config, SSL);
 convert_certs(RltvDir, #{ssl := SSL} = Config) ->
@@ -49,14 +35,6 @@ convert_certs(_RltvDir, Config) ->
 clear_certs(RltvDir, Config) ->
     clear_certs2(RltvDir, normalize_key_to_bin(Config)).
 
-clear_certs2(RltvDir, #{<<"connector">> := Connector} = _Config) when
-    is_map(Connector)
-->
-    %% TODO remove the 'connector' clause after dev/ee5.0 is merged back to master
-    %% The `connector` config layer will be removed.
-    %% for bridges with `connector` field. i.e. `mqtt_source` and `mqtt_sink`
-    OldSSL = maps:get(<<"ssl">>, Connector, undefined),
-    ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
 clear_certs2(RltvDir, #{<<"ssl">> := OldSSL} = _Config) ->
     ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
 clear_certs2(_RltvDir, _) ->
@@ -69,15 +47,10 @@ try_clear_certs(RltvDir, NewConf, OldConf) ->
         normalize_key_to_bin(OldConf)
     ).
 
-try_clear_certs2(RltvDir, #{<<"connector">> := NewConnector}, #{<<"connector">> := OldConnector}) ->
-    NewSSL = maps:get(<<"ssl">>, NewConnector, undefined),
-    OldSSL = maps:get(<<"ssl">>, OldConnector, undefined),
-    try_clear_certs2(RltvDir, NewSSL, OldSSL);
-try_clear_certs2(RltvDir, NewSSL, OldSSL) when is_map(NewSSL) andalso is_map(OldSSL) ->
-    ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL);
 try_clear_certs2(RltvDir, NewConf, OldConf) ->
-    ?SLOG(debug, #{msg => "unexpected_conf", path => RltvDir, new => NewConf, OldConf => OldConf}),
-    ok.
+    NewSSL = try_map_get(<<"ssl">>, NewConf, undefined),
+    OldSSL = try_map_get(<<"ssl">>, OldConf, undefined),
+    ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL).
 
 new_ssl_config(RltvDir, Config, SSL) ->
     case emqx_tls_lib:ensure_ssl_files(RltvDir, SSL) of
@@ -98,5 +71,12 @@ new_ssl_config(#{<<"ssl">> := _} = Config, NewSSL) ->
 new_ssl_config(Config, _NewSSL) ->
     Config.
 
-normalize_key_to_bin(Map) ->
+normalize_key_to_bin(undefined) ->
+    undefined;
+normalize_key_to_bin(Map) when is_map(Map) ->
     emqx_map_lib:binary_key_map(Map).
+
+try_map_get(_Key, undefined, Default) ->
+    Default;
+try_map_get(Key, Map, Default) when is_map(Map) ->
+    maps:get(Key, Map, Default).

From fa90638de20d3b186984ab1fb4ed77c0fd7e2132 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 14:28:29 +0100
Subject: [PATCH 215/232] chore: add description to apps

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src       | 1 +
 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index 2e6458682..2748c27a7 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -1,4 +1,5 @@
 {application, emqx_ee_bridge, [
+    {description, "EMQX Enterprise data bridges"},
     {vsn, "0.1.0"},
     {registered, []},
     {applications, [
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index c1b86d20b..1163e391c 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,4 +1,5 @@
 {application, emqx_ee_connector, [
+    {description, "EMQX Enterprise connectors"},
     {vsn, "0.1.0"},
     {registered, []},
     {applications, [

From 1084dd17235ba52453167f99a28e920b5fd5c5dc Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 14:30:19 +0100
Subject: [PATCH 216/232] ci: add --remove-orphans option to docker-compose up
 command

---
 scripts/ct/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 5a2cce036..d1e24558b 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -150,7 +150,7 @@ else
 fi
 
 # shellcheck disable=2086 # no quotes for F_OPTIONS
-docker-compose $F_OPTIONS up -d --build
+docker-compose $F_OPTIONS up -d --build --remove-orphans
 
 # /emqx is where the source dir is mounted to the Erlang container
 # in .ci/docker-compose-file/docker-compose.yaml

From 9a6901987f5c857bfbfb2042d998daffa5ffc999 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 28 Nov 2022 22:22:20 +0100
Subject: [PATCH 217/232] feat: add mqtt bridge config upgrade converter

---
 apps/emqx_bridge/src/emqx_bridge.erl          |   8 +-
 .../src/schema/emqx_bridge_mqtt_config.erl    | 116 ++++++++++++++++++
 .../src/schema/emqx_bridge_schema.erl         |   5 +-
 .../src/mqtt/emqx_connector_mqtt_schema.erl   |   1 +
 4 files changed, 128 insertions(+), 2 deletions(-)
 create mode 100644 apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index 3aff30859..44028e900 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -213,6 +213,7 @@ lookup(Id) ->
 lookup(Type, Name) ->
     RawConf = emqx:get_raw_config([bridges, Type, Name], #{}),
     lookup(Type, Name, RawConf).
+
 lookup(Type, Name, RawConf) ->
     case emqx_resource:get_instance(emqx_bridge_resource:resource_id(Type, Name)) of
         {error, not_found} ->
@@ -222,10 +223,15 @@ lookup(Type, Name, RawConf) ->
                 type => Type,
                 name => Name,
                 resource_data => Data,
-                raw_config => RawConf
+                raw_config => maybe_upgrade(Type, RawConf)
             }}
     end.
 
+maybe_upgrade(mqtt, Config) ->
+    emqx_bridge_mqtt_config:maybe_upgrade(Config);
+maybe_upgrade(_Other, Config) ->
+    Config.
+
 disable_enable(Action, BridgeType, BridgeName) when
     Action =:= disable; Action =:= enable
 ->
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl
new file mode 100644
index 000000000..2e4c17a25
--- /dev/null
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl
@@ -0,0 +1,116 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_mqtt_config).
+
+-export([
+    upgrade_pre_ee/1,
+    maybe_upgrade/1
+]).
+
+upgrade_pre_ee(undefined) ->
+    undefined;
+upgrade_pre_ee(Conf0) when is_map(Conf0) ->
+    maps:from_list(upgrade_pre_ee(maps:to_list(Conf0)));
+upgrade_pre_ee([]) ->
+    [];
+upgrade_pre_ee([{Name, Config} | Bridges]) ->
+    [{Name, maybe_upgrade(Config)} | upgrade_pre_ee(Bridges)].
+
+maybe_upgrade(#{<<"connector">> := _} = Config0) ->
+    Config1 = up(Config0),
+    Config = lists:map(fun binary_key/1, Config1),
+    maps:from_list(Config);
+maybe_upgrade(NewVersion) ->
+    NewVersion.
+
+binary_key({K, V}) ->
+    {atom_to_binary(K, utf8), V}.
+
+up(#{<<"connector">> := Connector} = Config) ->
+    Cn = fun(Key0, Default) ->
+        Key = atom_to_binary(Key0, utf8),
+        {Key0, maps:get(Key, Connector, Default)}
+    end,
+    Direction =
+        case maps:get(<<"direction">>, Config) of
+            <<"egress">> ->
+                {egress, egress(Config)};
+            <<"ingress">> ->
+                {ingress, ingress(Config)}
+        end,
+    Enable = maps:get(<<"enable">>, Config, true),
+    [
+        Cn(bridge_mode, false),
+        Cn(username, <<>>),
+        Cn(password, <<>>),
+        Cn(clean_start, true),
+        Cn(keepalive, <<"60s">>),
+        Cn(mode, <<"cluster_shareload">>),
+        Cn(proto_ver, <<"v4">>),
+        Cn(server, undefined),
+        Cn(retry_interval, <<"15s">>),
+        Cn(reconnect_interval, <<"15s">>),
+        Cn(ssl, default_ssl()),
+        {enable, Enable},
+        {resource_opts, default_resource_opts()},
+        Direction
+    ].
+
+default_ssl() ->
+    #{
+        <<"enable">> => false,
+        <<"verify">> => <<"verify_peer">>
+    }.
+
+default_resource_opts() ->
+    #{
+        <<"async_inflight_window">> => 100,
+        <<"auto_restart_interval">> => <<"60s">>,
+        <<"enable_queue">> => false,
+        <<"health_check_interval">> => <<"15s">>,
+        <<"max_queue_bytes">> => <<"1GB">>,
+        <<"query_mode">> => <<"sync">>,
+        <<"worker_pool_size">> => 16
+    }.
+
+egress(Config) ->
+    % <<"local">> % the old version has no 'local' config for egress
+    #{
+        <<"remote">> =>
+            #{
+                <<"topic">> => maps:get(<<"remote_topic">>, Config),
+                <<"qos">> => maps:get(<<"remote_qos">>, Config),
+                <<"retain">> => maps:get(<<"retain">>, Config),
+                <<"payload">> => maps:get(<<"payload">>, Config)
+            }
+    }.
+
+ingress(Config) ->
+    #{
+        <<"remote">> =>
+            #{
+                <<"qos">> => maps:get(<<"remote_qos">>, Config),
+                <<"topic">> => maps:get(<<"remote_topic">>, Config)
+            },
+        <<"local">> =>
+            #{
+                <<"payload">> => maps:get(<<"payload">>, Config),
+                <<"qos">> => maps:get(<<"local_qos">>, Config),
+                <<"retain">> => maps:get(<<"retain">>, Config, false)
+                %% <<"topic">> % th old version has no local topic for ingress
+            }
+    }.
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 8bfc1c78a..31fc3bcc1 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -101,7 +101,10 @@ fields(bridges) ->
         {mqtt,
             mk(
                 hoconsc:map(name, ref(emqx_bridge_mqtt_schema, "config")),
-                #{desc => ?DESC("bridges_mqtt")}
+                #{
+                    desc => ?DESC("bridges_mqtt"),
+                    converter => fun emqx_bridge_mqtt_config:upgrade_pre_ee/1
+                }
             )}
     ] ++ ee_fields_bridges();
 fields("metrics") ->
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index d77859dd7..50f5ee5df 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -109,6 +109,7 @@ fields("server_configs") ->
                 binary(),
                 #{
                     format => <<"password">>,
+                    sensitive => true,
                     desc => ?DESC("password")
                 }
             )},

From 3e679ceb572586576a6fba79f8049ad068ce73d0 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Fri, 18 Nov 2022 15:10:33 +0100
Subject: [PATCH 218/232] test: add basic tests for EE mysql bridge

---
 lib-ee/emqx_ee_bridge/docker-ct               |   1 +
 .../test/emqx_ee_bridge_mysql_SUITE.erl       | 260 ++++++++++++++++++
 2 files changed, 261 insertions(+)
 create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl

diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct
index 3be129d94..1548a3203 100644
--- a/lib-ee/emqx_ee_bridge/docker-ct
+++ b/lib-ee/emqx_ee_bridge/docker-ct
@@ -2,3 +2,4 @@ influxdb
 kafka
 mongo
 mongo_rs_sharded
+mysql
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
new file mode 100644
index 000000000..b6f876437
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
@@ -0,0 +1,260 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_bridge_mysql_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+% SQL definitions
+-define(SQL_BRIDGE,
+    "INSERT INTO mqtt_test(payload, arrived) "
+    "VALUES (${payload}, FROM_UNIXTIME(${timestamp}/1000))"
+).
+-define(SQL_CREATE_TABLE,
+    "CREATE TABLE IF NOT EXISTS mqtt_test (payload blob, arrived datetime NOT NULL) "
+    "DEFAULT CHARSET=utf8MB4;"
+).
+-define(SQL_DROP_TABLE, "DROP TABLE mqtt_test").
+-define(SQL_DELETE, "DELETE from mqtt_test").
+-define(SQL_SELECT, "SELECT payload FROM mqtt_test").
+
+% DB defaults
+-define(MYSQL_DATABASE, "mqtt").
+-define(MYSQL_USERNAME, "root").
+-define(MYSQL_PASSWORD, "public").
+
+%%------------------------------------------------------------------------------
+%% CT boilerplate
+%%------------------------------------------------------------------------------
+
+all() ->
+    [
+        {group, tcp},
+        {group, tls}
+        | (emqx_common_test_helpers:all(?MODULE) -- group_tests())
+    ].
+
+group_tests() ->
+    [
+        t_setup_via_config_and_publish,
+        t_setup_via_http_api_and_publish
+    ].
+
+groups() ->
+    [
+        {tcp, group_tests()},
+        {tls, group_tests()}
+    ].
+
+init_per_group(GroupType = tcp, Config) ->
+    MysqlHost = os:getenv("MYSQL_TCP_HOST", "mysql"),
+    MysqlPort = list_to_integer(os:getenv("MYSQL_TCP_PORT", "3306")),
+    common_init(GroupType, Config, MysqlHost, MysqlPort);
+init_per_group(GroupType = tls, Config) ->
+    MysqlHost = os:getenv("MYSQL_TLS_HOST", "mysql-tls"),
+    MysqlPort = list_to_integer(os:getenv("MYSQL_TLS_PORT", "3306")),
+    common_init(GroupType, Config, MysqlHost, MysqlPort).
+
+end_per_group(GroupType, Config) ->
+    drop_table_raw(GroupType, Config),
+    ok.
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_mgmt_api_test_util:end_suite(),
+    ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
+    ok.
+
+init_per_testcase(_Testcase, Config) ->
+    catch clear_table(Config),
+    delete_bridge(Config),
+    Config.
+
+end_per_testcase(_Testcase, Config) ->
+    catch clear_table(Config),
+    delete_bridge(Config),
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Helper fns
+%%------------------------------------------------------------------------------
+
+common_init(GroupType, Config0, MysqlHost, MysqlPort) ->
+    BridgeType = <<"mysql">>,
+    case emqx_common_test_helpers:is_tcp_server_available(MysqlHost, MysqlPort) of
+        true ->
+            % Ensure EE bridge module is loaded
+            _ = application:load(emqx_ee_bridge),
+            _ = emqx_ee_bridge:module_info(),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            emqx_mgmt_api_test_util:init_suite(),
+            {Name, MysqlConfig} = mysql_config(MysqlHost, MysqlPort, GroupType, BridgeType),
+            Config =
+                [
+                    {mysql_host, MysqlHost},
+                    {mysql_port, MysqlPort},
+                    {mysql_config, MysqlConfig},
+                    {mysql_bridge_type, BridgeType},
+                    {mysql_name, Name}
+                    | Config0
+                ],
+            create_table_raw(GroupType, Config),
+            Config;
+        false ->
+            {skip, no_mysql}
+    end.
+
+mysql_config(MysqlHost, MysqlPort0, GroupType, BridgeType) ->
+    MysqlPort = integer_to_list(MysqlPort0),
+    Server = MysqlHost ++ ":" ++ MysqlPort,
+    Name = iolist_to_binary(io_lib:format("~s-~s", [?MODULE, GroupType])),
+    SslEnabled =
+        case GroupType of
+            tcp -> "false";
+            tls -> "true"
+        end,
+    ConfigString =
+        io_lib:format(
+            "bridges.~s.~s {\n"
+            "  enable = true\n"
+            "  server = ~p\n"
+            "  database = ~p\n"
+            "  username = ~p\n"
+            "  password = ~p\n"
+            "  sql = ~p\n"
+            "  ssl = {\n"
+            "                 enable = ~w\n"
+            "  }\n"
+            "}",
+            [
+                BridgeType,
+                Name,
+                Server,
+                ?MYSQL_DATABASE,
+                ?MYSQL_USERNAME,
+                ?MYSQL_PASSWORD,
+                ?SQL_BRIDGE,
+                SslEnabled
+            ]
+        ),
+    {Name, parse_and_check(ConfigString, BridgeType, Name)}.
+
+parse_and_check(ConfigString, BridgeType, Name) ->
+    {ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
+    hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
+    #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf,
+    Config.
+
+create_bridge(Config) ->
+    BridgeType = ?config(mysql_bridge_type, Config),
+    Name = ?config(mysql_name, Config),
+    MysqlConfig = ?config(mysql_config, Config),
+    emqx_bridge:create(BridgeType, Name, MysqlConfig).
+
+delete_bridge(Config) ->
+    BridgeType = ?config(mysql_bridge_type, Config),
+    Name = ?config(mysql_name, Config),
+    emqx_bridge:remove(BridgeType, Name).
+
+create_bridge_http(Params) ->
+    Path = emqx_mgmt_api_test_util:api_path(["bridges"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of
+        {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
+        Error -> Error
+    end.
+
+query(Config, SqlQuery) ->
+    BridgeType = ?config(mysql_bridge_type, Config),
+    Name = ?config(mysql_name, Config),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
+    ?assertMatch({ok, connected}, emqx_resource:health_check(ResourceID)),
+    emqx_resource:simple_sync_query(ResourceID, {sql, SqlQuery}).
+
+send_message(Config, Payload) ->
+    Name = ?config(mysql_name, Config),
+    BridgeType = ?config(mysql_bridge_type, Config),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
+    ?assertMatch({ok, connected}, emqx_resource:health_check(ResourceID)),
+    % TODO: Check why we can't use send_message directly!
+    % BridgeID = emqx_bridge_resource:bridge_id(Type, Name),
+    % emqx_bridge:send_message(BridgeID, Payload).
+    emqx_resource:simple_sync_query(ResourceID, {send_message, Payload}).
+
+clear_table(Config) ->
+    query(Config, ?SQL_DELETE).
+
+get_payload(Config) ->
+    query(Config, ?SQL_SELECT).
+
+% We need to create and drop the test table outside of using bridges
+% since a bridge expects the table to exist when enabling it. We
+% therefore call the mysql module directly.
+connect_raw_and_run_sql(GroupType, Config, Sql) ->
+    Opts = [
+        {host, ?config(mysql_host, Config)},
+        {port, ?config(mysql_port, Config)},
+        {user, ?MYSQL_USERNAME},
+        {password, ?MYSQL_PASSWORD},
+        {database, ?MYSQL_DATABASE}
+    ],
+    SslOpts =
+        case GroupType of
+            tls ->
+                [{ssl, emqx_tls_lib:to_client_opts(#{enable => true})}];
+            tcp ->
+                []
+        end,
+    {ok, Pid} = mysql:start_link(Opts ++ SslOpts),
+    ok = mysql:query(Pid, Sql),
+    mysql:stop(Pid).
+
+create_table_raw(GroupType, Config) ->
+    connect_raw_and_run_sql(GroupType, Config, ?SQL_CREATE_TABLE).
+
+drop_table_raw(GroupType, Config) ->
+    connect_raw_and_run_sql(GroupType, Config, ?SQL_DROP_TABLE).
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_setup_via_config_and_publish(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    Val = integer_to_binary(erlang:unique_integer()),
+    ?assertMatch(ok, send_message(Config, #{payload => Val, timestamp => 1668602148000})),
+    ?assertMatch(
+        {ok, [<<"payload">>], [[Val]]},
+        get_payload(Config)
+    ),
+    ok.
+
+t_setup_via_http_api_and_publish(Config) ->
+    BridgeType = ?config(mysql_bridge_type, Config),
+    Name = ?config(mysql_name, Config),
+    MysqlConfig0 = ?config(mysql_config, Config),
+    MysqlConfig = MysqlConfig0#{
+        <<"name">> => Name,
+        <<"type">> => BridgeType
+    },
+    ?assertMatch(
+        {ok, _},
+        create_bridge_http(MysqlConfig)
+    ),
+    Val = integer_to_binary(erlang:unique_integer()),
+    ?assertMatch(ok, send_message(Config, #{payload => Val, timestamp => 1668602148000})),
+    ?assertMatch(
+        {ok, [<<"payload">>], [[Val]]},
+        get_payload(Config)
+    ),
+    ok.

From 96bff1d32e53c3ab1585b4f9b6e77666f8ecbd7a Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Fri, 25 Nov 2022 10:54:28 +0100
Subject: [PATCH 219/232] test: improve basic tests for EE mysql bridge

---
 .../src/emqx_connector_mysql.erl              |   9 +
 .../test/emqx_ee_bridge_mysql_SUITE.erl       | 210 ++++++++++++------
 2 files changed, 150 insertions(+), 69 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index b35f0b018..afc6fbdd2 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -19,6 +19,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -behaviour(emqx_resource).
 
@@ -398,6 +399,10 @@ on_sql_query(
     ?TRACE("QUERY", "mysql_connector_received", LogMeta),
     Worker = ecpool:get_client(PoolName),
     {ok, Conn} = ecpool_worker:client(Worker),
+    ?tp(
+        mysql_connector_send_query,
+        #{sql_or_key => SQLOrKey, data => Data}
+    ),
     try mysql:SQLFunc(Conn, SQLOrKey, Data, Timeout) of
         {error, disconnected} = Result ->
             ?SLOG(
@@ -427,6 +432,10 @@ on_sql_query(
             ),
             Result;
         Result ->
+            ?tp(
+                mysql_connector_query_return,
+                #{result => Result}
+            ),
             Result
     catch
         error:badarg ->
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
index b6f876437..753a9f3f9 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
@@ -9,6 +9,7 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 % SQL definitions
 -define(SQL_BRIDGE,
@@ -34,34 +35,68 @@
 
 all() ->
     [
-        {group, tcp},
-        {group, tls}
-        | (emqx_common_test_helpers:all(?MODULE) -- group_tests())
-    ].
-
-group_tests() ->
-    [
-        t_setup_via_config_and_publish,
-        t_setup_via_http_api_and_publish
+        {group, with_batch},
+        {group, without_batch}
     ].
 
 groups() ->
+    TCs = emqx_common_test_helpers:all(?MODULE),
     [
-        {tcp, group_tests()},
-        {tls, group_tests()}
+        {with_batch, [
+            {group, sync_query},
+            {group, async_query}
+        ]},
+        {without_batch, [
+            {group, sync_query},
+            {group, async_query}
+        ]},
+        {sync_query, [
+            {group, tcp},
+            {group, tls}
+        ]},
+        {async_query, [
+            {group, tcp},
+            {group, tls}
+        ]},
+        {tcp, TCs},
+        {tls, TCs}
     ].
 
-init_per_group(GroupType = tcp, Config) ->
+init_per_group(tcp, Config0) ->
     MysqlHost = os:getenv("MYSQL_TCP_HOST", "mysql"),
     MysqlPort = list_to_integer(os:getenv("MYSQL_TCP_PORT", "3306")),
-    common_init(GroupType, Config, MysqlHost, MysqlPort);
-init_per_group(GroupType = tls, Config) ->
+    Config = [
+        {mysql_host, MysqlHost},
+        {mysql_port, MysqlPort},
+        {enable_tls, false}
+        | Config0
+    ],
+    common_init(Config);
+init_per_group(tls, Config0) ->
     MysqlHost = os:getenv("MYSQL_TLS_HOST", "mysql-tls"),
     MysqlPort = list_to_integer(os:getenv("MYSQL_TLS_PORT", "3306")),
-    common_init(GroupType, Config, MysqlHost, MysqlPort).
+    Config = [
+        {mysql_host, MysqlHost},
+        {mysql_port, MysqlPort},
+        {enable_tls, true}
+        | Config0
+    ],
+    common_init(Config);
+init_per_group(sync_query, Config) ->
+    [{query_mode, sync} | Config];
+init_per_group(async_query, Config) ->
+    [{query_mode, async} | Config];
+init_per_group(with_batch, Config) ->
+    [{enable_batch, true} | Config];
+init_per_group(without_batch, Config) ->
+    [{enable_batch, false} | Config];
+init_per_group(_Group, Config) ->
+    Config.
 
-end_per_group(GroupType, Config) ->
-    drop_table_raw(GroupType, Config),
+end_per_group(Group, Config) when Group =:= tcp; Group =:= tls ->
+    connect_and_drop_table(Config),
+    ok;
+end_per_group(_Group, _Config) ->
     ok.
 
 init_per_suite(Config) ->
@@ -72,13 +107,16 @@ end_per_suite(_Config) ->
     ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
     ok.
 
-init_per_testcase(_Testcase, Config) ->
+init_per_testcase(_Testcase, Config0) ->
+    Config = [{mysql_direct_pid, connect_direct_mysql(Config0)} | Config0],
     catch clear_table(Config),
     delete_bridge(Config),
     Config.
 
 end_per_testcase(_Testcase, Config) ->
     catch clear_table(Config),
+    DirectPid = ?config(mysql_direct_pid, Config),
+    mysql:stop(DirectPid),
     delete_bridge(Config),
     ok.
 
@@ -86,8 +124,10 @@ end_per_testcase(_Testcase, Config) ->
 %% Helper fns
 %%------------------------------------------------------------------------------
 
-common_init(GroupType, Config0, MysqlHost, MysqlPort) ->
+common_init(Config0) ->
     BridgeType = <<"mysql">>,
+    MysqlHost = ?config(mysql_host, Config0),
+    MysqlPort = ?config(mysql_port, Config0),
     case emqx_common_test_helpers:is_tcp_server_available(MysqlHost, MysqlPort) of
         true ->
             % Ensure EE bridge module is loaded
@@ -95,31 +135,28 @@ common_init(GroupType, Config0, MysqlHost, MysqlPort) ->
             _ = emqx_ee_bridge:module_info(),
             ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
             emqx_mgmt_api_test_util:init_suite(),
-            {Name, MysqlConfig} = mysql_config(MysqlHost, MysqlPort, GroupType, BridgeType),
+            % Connect to mysql directly and create the table
+            connect_and_create_table(Config0),
+            {Name, MysqlConfig} = mysql_config(BridgeType, Config0),
             Config =
                 [
-                    {mysql_host, MysqlHost},
-                    {mysql_port, MysqlPort},
                     {mysql_config, MysqlConfig},
                     {mysql_bridge_type, BridgeType},
                     {mysql_name, Name}
                     | Config0
                 ],
-            create_table_raw(GroupType, Config),
             Config;
         false ->
             {skip, no_mysql}
     end.
 
-mysql_config(MysqlHost, MysqlPort0, GroupType, BridgeType) ->
-    MysqlPort = integer_to_list(MysqlPort0),
-    Server = MysqlHost ++ ":" ++ MysqlPort,
-    Name = iolist_to_binary(io_lib:format("~s-~s", [?MODULE, GroupType])),
-    SslEnabled =
-        case GroupType of
-            tcp -> "false";
-            tls -> "true"
-        end,
+mysql_config(BridgeType, Config) ->
+    MysqlPort = integer_to_list(?config(mysql_port, Config)),
+    Server = ?config(mysql_host, Config) ++ ":" ++ MysqlPort,
+    Name = atom_to_binary(?MODULE),
+    EnableBatch = ?config(enable_batch, Config),
+    QueryMode = ?config(query_mode, Config),
+    TlsEnabled = ?config(enable_tls, Config),
     ConfigString =
         io_lib:format(
             "bridges.~s.~s {\n"
@@ -129,6 +166,10 @@ mysql_config(MysqlHost, MysqlPort0, GroupType, BridgeType) ->
             "  username = ~p\n"
             "  password = ~p\n"
             "  sql = ~p\n"
+            "  resource_opts = {\n"
+            "    enable_batch = ~p\n"
+            "    query_mode = ~s\n"
+            "  }\n"
             "  ssl = {\n"
             "                 enable = ~w\n"
             "  }\n"
@@ -141,7 +182,9 @@ mysql_config(MysqlHost, MysqlPort0, GroupType, BridgeType) ->
                 ?MYSQL_USERNAME,
                 ?MYSQL_PASSWORD,
                 ?SQL_BRIDGE,
-                SslEnabled
+                EnableBatch,
+                QueryMode,
+                TlsEnabled
             ]
         ),
     {Name, parse_and_check(ConfigString, BridgeType, Name)}.
@@ -171,33 +214,19 @@ create_bridge_http(Params) ->
         Error -> Error
     end.
 
-query(Config, SqlQuery) ->
-    BridgeType = ?config(mysql_bridge_type, Config),
-    Name = ?config(mysql_name, Config),
-    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
-    ?assertMatch({ok, connected}, emqx_resource:health_check(ResourceID)),
-    emqx_resource:simple_sync_query(ResourceID, {sql, SqlQuery}).
-
 send_message(Config, Payload) ->
     Name = ?config(mysql_name, Config),
     BridgeType = ?config(mysql_bridge_type, Config),
+    BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name),
     ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
     ?assertMatch({ok, connected}, emqx_resource:health_check(ResourceID)),
-    % TODO: Check why we can't use send_message directly!
-    % BridgeID = emqx_bridge_resource:bridge_id(Type, Name),
-    % emqx_bridge:send_message(BridgeID, Payload).
-    emqx_resource:simple_sync_query(ResourceID, {send_message, Payload}).
-
-clear_table(Config) ->
-    query(Config, ?SQL_DELETE).
-
-get_payload(Config) ->
-    query(Config, ?SQL_SELECT).
+    emqx_bridge:send_message(BridgeID, Payload).
 
 % We need to create and drop the test table outside of using bridges
 % since a bridge expects the table to exist when enabling it. We
-% therefore call the mysql module directly.
-connect_raw_and_run_sql(GroupType, Config, Sql) ->
+% therefore call the mysql module directly, in addition to using it
+% for querying the DB directly.
+connect_direct_mysql(Config) ->
     Opts = [
         {host, ?config(mysql_host, Config)},
         {port, ?config(mysql_port, Config)},
@@ -206,21 +235,34 @@ connect_raw_and_run_sql(GroupType, Config, Sql) ->
         {database, ?MYSQL_DATABASE}
     ],
     SslOpts =
-        case GroupType of
-            tls ->
+        case ?config(enable_tls, Config) of
+            true ->
                 [{ssl, emqx_tls_lib:to_client_opts(#{enable => true})}];
-            tcp ->
+            false ->
                 []
         end,
     {ok, Pid} = mysql:start_link(Opts ++ SslOpts),
-    ok = mysql:query(Pid, Sql),
-    mysql:stop(Pid).
+    Pid.
 
-create_table_raw(GroupType, Config) ->
-    connect_raw_and_run_sql(GroupType, Config, ?SQL_CREATE_TABLE).
+% These funs connect and then stop the mysql connection
+connect_and_create_table(Config) ->
+    DirectPid = connect_direct_mysql(Config),
+    ok = mysql:query(DirectPid, ?SQL_CREATE_TABLE),
+    mysql:stop(DirectPid).
 
-drop_table_raw(GroupType, Config) ->
-    connect_raw_and_run_sql(GroupType, Config, ?SQL_DROP_TABLE).
+connect_and_drop_table(Config) ->
+    DirectPid = connect_direct_mysql(Config),
+    ok = mysql:query(DirectPid, ?SQL_DROP_TABLE),
+    mysql:stop(DirectPid).
+
+% These funs expects a connection to already exist
+clear_table(Config) ->
+    DirectPid = ?config(mysql_direct_pid, Config),
+    ok = mysql:query(DirectPid, ?SQL_DELETE).
+
+get_payload(Config) ->
+    DirectPid = ?config(mysql_direct_pid, Config),
+    mysql:query(DirectPid, ?SQL_SELECT).
 
 %%------------------------------------------------------------------------------
 %% Testcases
@@ -232,10 +274,25 @@ t_setup_via_config_and_publish(Config) ->
         create_bridge(Config)
     ),
     Val = integer_to_binary(erlang:unique_integer()),
-    ?assertMatch(ok, send_message(Config, #{payload => Val, timestamp => 1668602148000})),
-    ?assertMatch(
-        {ok, [<<"payload">>], [[Val]]},
-        get_payload(Config)
+    SentData = #{payload => Val, timestamp => 1668602148000},
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertEqual(ok, send_message(Config, SentData)),
+                #{?snk_kind := mysql_connector_query_return},
+                10_000
+            ),
+            ?assertMatch(
+                {ok, [<<"payload">>], [[Val]]},
+                get_payload(Config)
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(mysql_connector_query_return, Trace0),
+            ?assertMatch([#{result := ok}], Trace),
+            ok
+        end
     ),
     ok.
 
@@ -252,9 +309,24 @@ t_setup_via_http_api_and_publish(Config) ->
         create_bridge_http(MysqlConfig)
     ),
     Val = integer_to_binary(erlang:unique_integer()),
-    ?assertMatch(ok, send_message(Config, #{payload => Val, timestamp => 1668602148000})),
-    ?assertMatch(
-        {ok, [<<"payload">>], [[Val]]},
-        get_payload(Config)
+    SentData = #{payload => Val, timestamp => 1668602148000},
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertEqual(ok, send_message(Config, SentData)),
+                #{?snk_kind := mysql_connector_query_return},
+                10_000
+            ),
+            ?assertMatch(
+                {ok, [<<"payload">>], [[Val]]},
+                get_payload(Config)
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(mysql_connector_query_return, Trace0),
+            ?assertMatch([#{result := ok}], Trace),
+            ok
+        end
     ),
     ok.

From eb62192838768458e85cef38c348eb23d7684ce8 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Mon, 28 Nov 2022 13:45:52 +0100
Subject: [PATCH 220/232] test: expand EE mysql bridge test with toxiproxy

---
 .../docker-compose-toxiproxy.yaml             |  2 +
 .ci/docker-compose-file/toxiproxy.json        | 12 +++
 lib-ee/emqx_ee_bridge/docker-ct               |  1 +
 .../test/emqx_ee_bridge_mysql_SUITE.erl       | 81 ++++++++++++++-----
 scripts/ct/run.sh                             |  6 +-
 5 files changed, 78 insertions(+), 24 deletions(-)

diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
index 924c9e6ae..66e7ec308 100644
--- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
+++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
@@ -13,6 +13,8 @@ services:
       - 8474:8474
       - 8086:8086
       - 8087:8087
+      - 13306:3306
+      - 13307:3307
     command:
       - "-host=0.0.0.0"
       - "-config=/config/toxiproxy.json"
diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json
index 176b35d36..2d3a30b6b 100644
--- a/.ci/docker-compose-file/toxiproxy.json
+++ b/.ci/docker-compose-file/toxiproxy.json
@@ -10,5 +10,17 @@
     "listen": "0.0.0.0:8087",
     "upstream": "influxdb_tls:8086",
     "enabled": true
+  },
+  {
+    "name": "mysql_tcp",
+    "listen": "0.0.0.0:3306",
+    "upstream": "mysql:3306",
+    "enabled": true
+  },
+  {
+    "name": "mysql_tls",
+    "listen": "0.0.0.0:3307",
+    "upstream": "mysql-tls:3306",
+    "enabled": true
   }
 ]
diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct
index 1548a3203..94f9379df 100644
--- a/lib-ee/emqx_ee_bridge/docker-ct
+++ b/lib-ee/emqx_ee_bridge/docker-ct
@@ -1,3 +1,4 @@
+toxiproxy
 influxdb
 kafka
 mongo
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
index 753a9f3f9..655fee6e5 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
@@ -63,22 +63,24 @@ groups() ->
     ].
 
 init_per_group(tcp, Config0) ->
-    MysqlHost = os:getenv("MYSQL_TCP_HOST", "mysql"),
+    MysqlHost = os:getenv("MYSQL_TCP_HOST", "toxiproxy"),
     MysqlPort = list_to_integer(os:getenv("MYSQL_TCP_PORT", "3306")),
     Config = [
         {mysql_host, MysqlHost},
         {mysql_port, MysqlPort},
-        {enable_tls, false}
+        {enable_tls, false},
+        {proxy_name, "mysql_tcp"}
         | Config0
     ],
     common_init(Config);
 init_per_group(tls, Config0) ->
-    MysqlHost = os:getenv("MYSQL_TLS_HOST", "mysql-tls"),
-    MysqlPort = list_to_integer(os:getenv("MYSQL_TLS_PORT", "3306")),
+    MysqlHost = os:getenv("MYSQL_TLS_HOST", "toxiproxy"),
+    MysqlPort = list_to_integer(os:getenv("MYSQL_TLS_PORT", "3307")),
     Config = [
         {mysql_host, MysqlHost},
         {mysql_port, MysqlPort},
-        {enable_tls, true}
+        {enable_tls, true},
+        {proxy_name, "mysql_tls"}
         | Config0
     ],
     common_init(Config);
@@ -95,6 +97,9 @@ init_per_group(_Group, Config) ->
 
 end_per_group(Group, Config) when Group =:= tcp; Group =:= tls ->
     connect_and_drop_table(Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
     ok;
 end_per_group(_Group, _Config) ->
     ok.
@@ -107,16 +112,18 @@ end_per_suite(_Config) ->
     ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
     ok.
 
-init_per_testcase(_Testcase, Config0) ->
-    Config = [{mysql_direct_pid, connect_direct_mysql(Config0)} | Config0],
-    catch clear_table(Config),
+init_per_testcase(_Testcase, Config) ->
+    %    Config = [{mysql_direct_pid, connect_direct_mysql(Config0)} | Config0],
+    connect_and_clear_table(Config),
     delete_bridge(Config),
     Config.
 
 end_per_testcase(_Testcase, Config) ->
-    catch clear_table(Config),
-    DirectPid = ?config(mysql_direct_pid, Config),
-    mysql:stop(DirectPid),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    connect_and_clear_table(Config),
+    ok = snabbkaffe:stop(),
     delete_bridge(Config),
     ok.
 
@@ -130,6 +137,10 @@ common_init(Config0) ->
     MysqlPort = ?config(mysql_port, Config0),
     case emqx_common_test_helpers:is_tcp_server_available(MysqlHost, MysqlPort) of
         true ->
+            % Setup toxiproxy
+            ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
+            ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
+            emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
             % Ensure EE bridge module is loaded
             _ = application:load(emqx_ee_bridge),
             _ = emqx_ee_bridge:module_info(),
@@ -142,7 +153,9 @@ common_init(Config0) ->
                 [
                     {mysql_config, MysqlConfig},
                     {mysql_bridge_type, BridgeType},
-                    {mysql_name, Name}
+                    {mysql_name, Name},
+                    {proxy_host, ProxyHost},
+                    {proxy_port, ProxyPort}
                     | Config0
                 ],
             Config;
@@ -171,7 +184,7 @@ mysql_config(BridgeType, Config) ->
             "    query_mode = ~s\n"
             "  }\n"
             "  ssl = {\n"
-            "                 enable = ~w\n"
+            "    enable = ~w\n"
             "  }\n"
             "}",
             [
@@ -255,14 +268,16 @@ connect_and_drop_table(Config) ->
     ok = mysql:query(DirectPid, ?SQL_DROP_TABLE),
     mysql:stop(DirectPid).
 
-% These funs expects a connection to already exist
-clear_table(Config) ->
-    DirectPid = ?config(mysql_direct_pid, Config),
-    ok = mysql:query(DirectPid, ?SQL_DELETE).
+connect_and_clear_table(Config) ->
+    DirectPid = connect_direct_mysql(Config),
+    ok = mysql:query(DirectPid, ?SQL_DELETE),
+    mysql:stop(DirectPid).
 
-get_payload(Config) ->
-    DirectPid = ?config(mysql_direct_pid, Config),
-    mysql:query(DirectPid, ?SQL_SELECT).
+connect_and_get_payload(Config) ->
+    DirectPid = connect_direct_mysql(Config),
+    Result = mysql:query(DirectPid, ?SQL_SELECT),
+    mysql:stop(DirectPid),
+    Result.
 
 %%------------------------------------------------------------------------------
 %% Testcases
@@ -284,7 +299,7 @@ t_setup_via_config_and_publish(Config) ->
             ),
             ?assertMatch(
                 {ok, [<<"payload">>], [[Val]]},
-                get_payload(Config)
+                connect_and_get_payload(Config)
             ),
             ok
         end,
@@ -319,7 +334,7 @@ t_setup_via_http_api_and_publish(Config) ->
             ),
             ?assertMatch(
                 {ok, [<<"payload">>], [[Val]]},
-                get_payload(Config)
+                connect_and_get_payload(Config)
             ),
             ok
         end,
@@ -330,3 +345,25 @@ t_setup_via_http_api_and_publish(Config) ->
         end
     ),
     ok.
+
+t_get_status(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyName = ?config(proxy_name, Config),
+
+    Name = ?config(mysql_name, Config),
+    BridgeType = ?config(mysql_bridge_type, Config),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
+
+    ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)),
+    emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+        ?assertMatch(
+            {ok, Status} when Status =:= disconnected orelse Status =:= connecting,
+            emqx_resource_manager:health_check(ResourceID)
+        )
+    end),
+    ok.
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index d1e24558b..18dfb2525 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -92,9 +92,11 @@ for dep in ${CT_DEPS}; do
         erlang24)
             FILES+=( '.ci/docker-compose-file/docker-compose.yaml' )
             ;;
+        toxiproxy)
+            FILES+=( '.ci/docker-compose-file/docker-compose-toxiproxy.yaml' )
+            ;;
         influxdb)
-            FILES+=( '.ci/docker-compose-file/docker-compose-toxiproxy.yaml'
-                     '.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml'
+            FILES+=( '.ci/docker-compose-file/docker-compose-influxdb-tcp.yaml'
                      '.ci/docker-compose-file/docker-compose-influxdb-tls.yaml' )
             ;;
         mongo)

From 5461311edc1ea34225297ed8cd861ea54f6d1d4b Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Wed, 30 Nov 2022 14:28:50 +0100
Subject: [PATCH 221/232] test: more EE mysql bridge tests

---
 .../src/emqx_connector_mysql.erl              | 10 ++++-
 .../test/emqx_ee_bridge_mysql_SUITE.erl       | 45 +++++++++++++++++--
 2 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index afc6fbdd2..fc3068c66 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -119,8 +119,14 @@ on_start(
     Prepares = parse_prepare_sql(Config),
     State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares),
     case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
-        ok -> {ok, init_prepare(State)};
-        {error, Reason} -> {error, Reason}
+        ok ->
+            {ok, init_prepare(State)};
+        {error, Reason} ->
+            ?tp(
+                mysql_connector_start_failed,
+                #{error => Reason}
+            ),
+            {error, Reason}
     end.
 
 on_stop(InstId, #{poolname := PoolName}) ->
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
index 655fee6e5..0d8f68b39 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
@@ -113,7 +113,6 @@ end_per_suite(_Config) ->
     ok.
 
 init_per_testcase(_Testcase, Config) ->
-    %    Config = [{mysql_direct_pid, connect_direct_mysql(Config0)} | Config0],
     connect_and_clear_table(Config),
     delete_bridge(Config),
     Config.
@@ -231,8 +230,6 @@ send_message(Config, Payload) ->
     Name = ?config(mysql_name, Config),
     BridgeType = ?config(mysql_bridge_type, Config),
     BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name),
-    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
-    ?assertMatch({ok, connected}, emqx_resource:health_check(ResourceID)),
     emqx_bridge:send_message(BridgeID, Payload).
 
 % We need to create and drop the test table outside of using bridges
@@ -367,3 +364,45 @@ t_get_status(Config) ->
         )
     end),
     ok.
+
+t_create_disconnected(Config) ->
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyName = ?config(proxy_name, Config),
+    ?check_trace(
+        emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+            ?assertMatch({ok, _}, create_bridge(Config))
+        end),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{error := {start_pool_failed, _, _}}],
+                ?of_kind(mysql_connector_start_failed, Trace)
+            ),
+            ok
+        end
+    ),
+    ok.
+
+t_write_failure(Config) ->
+    ProxyName = ?config(proxy_name, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    QueryMode = ?config(query_mode, Config),
+    {ok, _} = create_bridge(Config),
+    Val = integer_to_binary(erlang:unique_integer()),
+    SentData = #{payload => Val, timestamp => 1668602148000},
+    ?check_trace(
+        emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+            send_message(Config, SentData)
+        end),
+        fun(Result, _Trace) ->
+            case QueryMode of
+                sync ->
+                    ?assertMatch({error, {resource_error, _}}, Result);
+                async ->
+                    ?assertEqual(ok, Result)
+            end,
+            ok
+        end
+    ),
+    ok.

From 499a32ce36bd34e50cccd03147e7883918af9540 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 1 Dec 2022 16:38:03 +0100
Subject: [PATCH 222/232] test: remove unnecessary async tests in EE mysql
 bridge

The async test cases is not needed since the mysql connector uses the
always_sync callback mode.
---
 .../test/emqx_ee_bridge_mysql_SUITE.erl       | 20 +++----------------
 1 file changed, 3 insertions(+), 17 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
index 0d8f68b39..292c02580 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl
@@ -43,21 +43,15 @@ groups() ->
     TCs = emqx_common_test_helpers:all(?MODULE),
     [
         {with_batch, [
-            {group, sync_query},
-            {group, async_query}
+            {group, sync_query}
         ]},
         {without_batch, [
-            {group, sync_query},
-            {group, async_query}
+            {group, sync_query}
         ]},
         {sync_query, [
             {group, tcp},
             {group, tls}
         ]},
-        {async_query, [
-            {group, tcp},
-            {group, tls}
-        ]},
         {tcp, TCs},
         {tls, TCs}
     ].
@@ -86,8 +80,6 @@ init_per_group(tls, Config0) ->
     common_init(Config);
 init_per_group(sync_query, Config) ->
     [{query_mode, sync} | Config];
-init_per_group(async_query, Config) ->
-    [{query_mode, async} | Config];
 init_per_group(with_batch, Config) ->
     [{enable_batch, true} | Config];
 init_per_group(without_batch, Config) ->
@@ -387,7 +379,6 @@ t_write_failure(Config) ->
     ProxyName = ?config(proxy_name, Config),
     ProxyPort = ?config(proxy_port, Config),
     ProxyHost = ?config(proxy_host, Config),
-    QueryMode = ?config(query_mode, Config),
     {ok, _} = create_bridge(Config),
     Val = integer_to_binary(erlang:unique_integer()),
     SentData = #{payload => Val, timestamp => 1668602148000},
@@ -396,12 +387,7 @@ t_write_failure(Config) ->
             send_message(Config, SentData)
         end),
         fun(Result, _Trace) ->
-            case QueryMode of
-                sync ->
-                    ?assertMatch({error, {resource_error, _}}, Result);
-                async ->
-                    ?assertEqual(ok, Result)
-            end,
+            ?assertMatch({error, {resource_error, _}}, Result),
             ok
         end
     ),

From 7adb539ae1c5f8f2deaad69a043244a695ae47dd Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 22:40:23 +0100
Subject: [PATCH 223/232] chore: pin hocon 0.31.2

---
 apps/emqx/rebar.config             | 2 +-
 lib-ee/emqx_ee_bridge/rebar.config | 2 +-
 mix.exs                            | 2 +-
 rebar.config                       | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index e50264693..d13fda30a 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.6"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config
index bfd1c957e..9119b052d 100644
--- a/lib-ee/emqx_ee_bridge/rebar.config
+++ b/lib-ee/emqx_ee_bridge/rebar.config
@@ -1,5 +1,5 @@
 {erl_opts, [debug_info]}.
-{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}
+{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}}
        , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
diff --git a/mix.exs b/mix.exs
index c7d9ae856..a3c18842c 100644
--- a/mix.exs
+++ b/mix.exs
@@ -67,7 +67,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.30.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.31.2", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
diff --git a/rebar.config b/rebar.config
index 369da655a..687f49cea 100644
--- a/rebar.config
+++ b/rebar.config
@@ -67,7 +67,7 @@
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

From 9ea22d062da588b463190ae819413699dcd39823 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 30 Nov 2022 23:19:35 +0100
Subject: [PATCH 224/232] refactor: make all bridges optional (required =
 false)

---
 .../src/schema/emqx_bridge_mqtt_config.erl    |  2 ++
 .../src/schema/emqx_bridge_schema.erl         |  6 ++++-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  | 25 +++++++++++++++----
 3 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl
index 2e4c17a25..997337c9d 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_config.erl
@@ -14,6 +14,8 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
+%% @doc This module was created to convert old version (from v5.0.0 to v5.0.11)
+%% mqtt connector configs to newer version (developed for enterprise edition).
 -module(emqx_bridge_mqtt_config).
 
 -export([
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 31fc3bcc1..756a8347d 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -96,13 +96,17 @@ fields(bridges) ->
         {webhook,
             mk(
                 hoconsc:map(name, ref(emqx_bridge_webhook_schema, "config")),
-                #{desc => ?DESC("bridges_webhook")}
+                #{
+                    desc => ?DESC("bridges_webhook"),
+                    required => false
+                }
             )},
         {mqtt,
             mk(
                 hoconsc:map(name, ref(emqx_bridge_mqtt_schema, "config")),
                 #{
                     desc => ?DESC("bridges_mqtt"),
+                    required => false,
                     converter => fun emqx_bridge_mqtt_config:upgrade_pre_ee/1
                 }
             )}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 96efee066..e0d362f5e 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -62,17 +62,26 @@ fields(bridges) ->
         {kafka,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_kafka, "config")),
-                #{desc => <<"EMQX Enterprise Config">>}
+                #{
+                    desc => <<"Kafka Bridge Config">>,
+                    required => false
+                }
             )},
         {hstreamdb,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")),
-                #{desc => <<"EMQX Enterprise Config">>}
+                #{
+                    desc => <<"HStreamDB Bridge Config">>,
+                    required => false
+                }
             )},
         {mysql,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_mysql, "config")),
-                #{desc => <<"EMQX Enterprise Config">>}
+                #{
+                    desc => <<"MySQL Bridge Config">>,
+                    required => false
+                }
             )}
     ] ++ mongodb_structs() ++ influxdb_structs().
 
@@ -81,7 +90,10 @@ mongodb_structs() ->
         {Type,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_mongodb, Type)),
-                #{desc => <<"EMQX Enterprise Config">>}
+                #{
+                    desc => <<"MongoDB Bridge Config">>,
+                    required => false
+                }
             )}
      || Type <- [mongodb_rs, mongodb_sharded, mongodb_single]
     ].
@@ -91,7 +103,10 @@ influxdb_structs() ->
         {Protocol,
             mk(
                 hoconsc:map(name, ref(emqx_ee_bridge_influxdb, Protocol)),
-                #{desc => <<"EMQX Enterprise Config">>}
+                #{
+                    desc => <<"InfluxDB Bridge Config">>,
+                    required => false
+                }
             )}
      || Protocol <- [
             %% influxdb_udp,

From eb017ab034cc2118f43cfc04338d151bdb3736f5 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 1 Dec 2022 16:11:41 +0100
Subject: [PATCH 225/232] test: add test case to cover mqtt bridge config
 upgrade

---
 .../src/schema/emqx_bridge_webhook_schema.erl |   5 +-
 .../test/emqx_bridge_mqtt_config_tests.erl    | 229 ++++++++++++++++++
 .../src/mqtt/emqx_connector_mqtt_schema.erl   |  27 ++-
 3 files changed, 252 insertions(+), 9 deletions(-)
 create mode 100644 apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl

diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
index 0f692f195..d270fc91e 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl
@@ -69,7 +69,10 @@ request_config() ->
         {local_topic,
             mk(
                 binary(),
-                #{desc => ?DESC("config_local_topic")}
+                #{
+                    desc => ?DESC("config_local_topic"),
+                    required => false
+                }
             )},
         {method,
             mk(
diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl
new file mode 100644
index 000000000..fa3fff7d9
--- /dev/null
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_config_tests.erl
@@ -0,0 +1,229 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_mqtt_config_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+empty_config_test() ->
+    Conf1 = #{<<"bridges">> => #{}},
+    Conf2 = #{<<"bridges">> => #{<<"webhook">> => #{}}},
+    ?assertEqual(Conf1, check(Conf1)),
+    ?assertEqual(Conf2, check(Conf2)),
+    ok.
+
+%% ensure webhook config can be checked
+webhook_config_test() ->
+    Conf = parse(webhook_v5011_hocon()),
+    ?assertMatch(
+        #{
+            <<"bridges">> :=
+                #{
+                    <<"webhook">> := #{
+                        <<"the_name">> :=
+                            #{
+                                <<"method">> := get,
+                                <<"body">> := <<"${payload}">>
+                            }
+                    }
+                }
+        },
+        check(Conf)
+    ),
+    ok.
+
+up(#{<<"bridges">> := Bridges0} = Conf0) ->
+    Bridges = up(Bridges0),
+    Conf0#{<<"bridges">> := Bridges};
+up(#{<<"mqtt">> := MqttBridges0} = Bridges) ->
+    MqttBridges = emqx_bridge_mqtt_config:upgrade_pre_ee(MqttBridges0),
+    Bridges#{<<"mqtt">> := MqttBridges}.
+
+parse(HOCON) ->
+    {ok, Conf} = hocon:binary(HOCON),
+    Conf.
+
+mqtt_config_test_() ->
+    Conf0 = mqtt_v5011_hocon(),
+    Conf1 = mqtt_v5011_full_hocon(),
+    [
+        {Tag, fun() ->
+            Parsed = parse(Conf),
+            Upgraded = up(Parsed),
+            Checked = check(Upgraded),
+            assert_upgraded(Checked)
+        end}
+     || {Tag, Conf} <- [{"minimum", Conf0}, {"full", Conf1}]
+    ].
+
+assert_upgraded(#{<<"bridges">> := Bridges}) ->
+    assert_upgraded(Bridges);
+assert_upgraded(#{<<"mqtt">> := Mqtt}) ->
+    assert_upgraded(Mqtt);
+assert_upgraded(#{<<"bridge_one">> := Map}) ->
+    assert_upgraded1(Map);
+assert_upgraded(#{<<"bridge_two">> := Map}) ->
+    assert_upgraded1(Map).
+
+assert_upgraded1(Map) ->
+    ?assertNot(maps:is_key(<<"connector">>, Map)),
+    ?assertNot(maps:is_key(<<"direction">>, Map)),
+    ?assert(maps:is_key(<<"server">>, Map)),
+    ?assert(maps:is_key(<<"ssl">>, Map)).
+
+check(Conf) when is_map(Conf) ->
+    hocon_tconf:check_plain(emqx_bridge_schema, Conf).
+
+%% erlfmt-ignore
+%% this is config generated from v5.0.11
+webhook_v5011_hocon() ->
+"""
+bridges{
+  webhook {
+    the_name{
+      body = \"${payload}\"
+      connect_timeout = \"5s\"
+      enable_pipelining = 100
+      headers {\"content-type\" = \"application/json\"}
+      max_retries = 3
+      method = \"get\"
+      pool_size = 4
+      request_timeout = \"5s\"
+      ssl {enable = false, verify = \"verify_peer\"}
+      url = \"http://localhost:8080\"
+    }
+  }
+}
+""".
+
+%% erlfmt-ignore
+%% this is a generated from v5.0.11
+mqtt_v5011_hocon() ->
+"""
+bridges {
+  mqtt {
+    bridge_one {
+      connector {
+        bridge_mode = false
+        clean_start = true
+        keepalive = \"60s\"
+        mode = cluster_shareload
+        proto_ver = \"v4\"
+        server = \"localhost:1883\"
+        ssl {enable = false, verify = \"verify_peer\"}
+      }
+      direction = egress
+      enable = true
+      payload = \"${payload}\"
+      remote_qos = 1
+      remote_topic = \"tttttttttt\"
+      retain = false
+    }
+    bridge_two {
+      connector {
+        bridge_mode = false
+        clean_start = true
+        keepalive = \"60s\"
+        mode = \"cluster_shareload\"
+        proto_ver = \"v4\"
+        server = \"localhost:1883\"
+        ssl {enable = false, verify = \"verify_peer\"}
+      }
+      direction = ingress
+      enable = true
+      local_qos = 1
+      payload = \"${payload}\"
+      remote_qos = 1
+      remote_topic = \"tttttttt/#\"
+      retain = false
+    }
+  }
+}
+""".
+
+%% erlfmt-ignore
+%% a more complete version
+mqtt_v5011_full_hocon() ->
+"""
+bridges {
+  mqtt {
+    bridge_one {
+      connector {
+        bridge_mode = false
+        clean_start = true
+        keepalive = \"60s\"
+        max_inflight = 32
+        mode = \"cluster_shareload\"
+        password = \"\"
+        proto_ver = \"v5\"
+        reconnect_interval = \"15s\"
+        replayq {offload = false, seg_bytes = \"100MB\"}
+        retry_interval = \"12s\"
+        server = \"localhost:1883\"
+        ssl {
+          ciphers = \"\"
+          depth = 10
+          enable = false
+          reuse_sessions = true
+          secure_renegotiate = true
+          user_lookup_fun = \"emqx_tls_psk:lookup\"
+          verify = \"verify_peer\"
+          versions = [\"tlsv1.3\", \"tlsv1.2\", \"tlsv1.1\", \"tlsv1\"]
+        }
+        username = \"\"
+      }
+      direction = \"ingress\"
+      enable = true
+      local_qos = 1
+      payload = \"${payload}\"
+      remote_qos = 1
+      remote_topic = \"tttt/a\"
+      retain = false
+    }
+    bridge_two {
+      connector {
+        bridge_mode = false
+        clean_start = true
+        keepalive = \"60s\"
+        max_inflight = 32
+        mode = \"cluster_shareload\"
+        password = \"\"
+        proto_ver = \"v4\"
+        reconnect_interval = \"15s\"
+        replayq {offload = false, seg_bytes = \"100MB\"}
+        retry_interval = \"44s\"
+        server = \"localhost:1883\"
+        ssl {
+          ciphers = \"\"
+          depth = 10
+          enable = false
+          reuse_sessions = true
+          secure_renegotiate = true
+          user_lookup_fun = \"emqx_tls_psk:lookup\"
+          verify = verify_peer
+          versions = [\"tlsv1.3\", \"tlsv1.2\", \"tlsv1.1\", \"tlsv1\"]
+        }
+        username = \"\"
+      }
+      direction = egress
+      enable = true
+      payload = \"${payload.x}\"
+      remote_qos = 1
+      remote_topic = \"remotetopic/1\"
+      retain = false
+    }
+  }
+}
+""".
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index 50f5ee5df..93bd846e4 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -42,17 +42,17 @@ fields("config") ->
         [
             {"ingress",
                 mk(
-                    hoconsc:union([none, ref(?MODULE, "ingress")]),
+                    ref(?MODULE, "ingress"),
                     #{
-                        default => undefined,
+                        required => {false, recursively},
                         desc => ?DESC("ingress_desc")
                     }
                 )},
             {"egress",
                 mk(
-                    hoconsc:union([none, ref(?MODULE, "egress")]),
+                    ref(?MODULE, "egress"),
                     #{
-                        default => undefined,
+                        required => {false, recursively},
                         desc => ?DESC("egress_desc")
                     }
                 )}
@@ -147,7 +147,10 @@ fields("ingress") ->
         {"local",
             mk(
                 ref(?MODULE, "ingress_local"),
-                #{desc => ?DESC(emqx_connector_mqtt_schema, "ingress_local")}
+                #{
+                    desc => ?DESC(emqx_connector_mqtt_schema, "ingress_local"),
+                    is_required => false
+                }
             )}
     ];
 fields("ingress_remote") ->
@@ -177,7 +180,8 @@ fields("ingress_local") ->
                 binary(),
                 #{
                     validator => fun emqx_schema:non_empty_string/1,
-                    desc => ?DESC("ingress_local_topic")
+                    desc => ?DESC("ingress_local_topic"),
+                    required => false
                 }
             )},
         {qos,
@@ -210,12 +214,18 @@ fields("egress") ->
         {"local",
             mk(
                 ref(?MODULE, "egress_local"),
-                #{desc => ?DESC(emqx_connector_mqtt_schema, "egress_local")}
+                #{
+                    desc => ?DESC(emqx_connector_mqtt_schema, "egress_local"),
+                    required => false
+                }
             )},
         {"remote",
             mk(
                 ref(?MODULE, "egress_remote"),
-                #{desc => ?DESC(emqx_connector_mqtt_schema, "egress_remote")}
+                #{
+                    desc => ?DESC(emqx_connector_mqtt_schema, "egress_remote"),
+                    required => true
+                }
             )}
     ];
 fields("egress_local") ->
@@ -225,6 +235,7 @@ fields("egress_local") ->
                 binary(),
                 #{
                     desc => ?DESC("egress_local_topic"),
+                    required => false,
                     validator => fun emqx_schema:non_empty_string/1
                 }
             )}

From 64effacaa7f37174c89b815e7515a8a65469119f Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 6 Dec 2022 16:31:16 +0100
Subject: [PATCH 226/232] chore: pin dashboard v1.1.3-sync-code

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 8cec47020..7ccb91c3c 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.1.2
+export EMQX_DASHBOARD_VERSION ?= v1.1.3-sync-code
 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.5
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1

From 31098d6c67b3c1b2252c32cb81d9d50be5489d25 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 6 Dec 2022 19:03:32 +0100
Subject: [PATCH 227/232] test: fix bridge api tests, metrics is now a separate
 api

---
 .../test/emqx_bridge_mqtt_SUITE.erl           | 47 +++++++++++--------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
index 84152efc6..c907205f1 100644
--- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl
@@ -313,9 +313,10 @@ t_ingress_mqtt_bridge_with_rules(_) ->
     ),
     %% and also the rule should be matched, with matched + 1:
     {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
+    {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []),
+    ?assertMatch(#{<<"id">> := RuleId}, jsx:decode(Rule1)),
     ?assertMatch(
         #{
-            <<"id">> := RuleId,
             <<"metrics">> := #{
                 <<"matched">> := 1,
                 <<"passed">> := 1,
@@ -332,8 +333,9 @@ t_ingress_mqtt_bridge_with_rules(_) ->
                 <<"actions.failed.unknown">> := 0
             }
         },
-        jsx:decode(Rule1)
+        jsx:decode(Metrics)
     ),
+
     %% we also check if the actions of the rule is triggered
     ?assertMatch(
         #{
@@ -429,24 +431,29 @@ t_egress_mqtt_bridge_with_rules(_) ->
     timer:sleep(100),
     emqx:publish(emqx_message:make(RuleTopic, Payload2)),
     {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
-    #{
-        <<"id">> := RuleId,
-        <<"metrics">> := #{
-            <<"matched">> := 1,
-            <<"passed">> := 1,
-            <<"failed">> := 0,
-            <<"failed.exception">> := 0,
-            <<"failed.no_result">> := 0,
-            <<"matched.rate">> := _,
-            <<"matched.rate.max">> := _,
-            <<"matched.rate.last5m">> := _,
-            <<"actions.total">> := 1,
-            <<"actions.success">> := 1,
-            <<"actions.failed">> := 0,
-            <<"actions.failed.out_of_service">> := 0,
-            <<"actions.failed.unknown">> := 0
-        }
-    } = jsx:decode(Rule1),
+    ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, jsx:decode(Rule1)),
+    {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []),
+    ?assertMatch(
+        #{
+            <<"metrics">> := #{
+                <<"matched">> := 1,
+                <<"passed">> := 1,
+                <<"failed">> := 0,
+                <<"failed.exception">> := 0,
+                <<"failed.no_result">> := 0,
+                <<"matched.rate">> := _,
+                <<"matched.rate.max">> := _,
+                <<"matched.rate.last5m">> := _,
+                <<"actions.total">> := 1,
+                <<"actions.success">> := 1,
+                <<"actions.failed">> := 0,
+                <<"actions.failed.out_of_service">> := 0,
+                <<"actions.failed.unknown">> := 0
+            }
+        },
+        jsx:decode(Metrics)
+    ),
+
     %% we should receive a message on the "remote" broker, with specified topic
     ?assert(
         receive

From f3b069a0d9ae212d5c7f9e542a4d939afea4e47d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 7 Dec 2022 15:50:50 +0100
Subject: [PATCH 228/232] test: fix config load for lib-ee tests

---
 .ci/docker-compose-file/.env                |  1 +
 Makefile                                    |  8 +--
 apps/emqx/test/emqx_common_test_helpers.erl | 58 ++++++++++++---------
 mix.exs                                     |  4 +-
 scripts/merge-config.escript                |  2 +-
 5 files changed, 41 insertions(+), 32 deletions(-)

diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env
index ae3d12c64..bd925e224 100644
--- a/.ci/docker-compose-file/.env
+++ b/.ci/docker-compose-file/.env
@@ -3,6 +3,7 @@ REDIS_TAG=6
 MONGO_TAG=5
 PGSQL_TAG=13
 LDAP_TAG=2.4.50
+INFLUXDB_TAG=2.5.0
 
 TARGET=emqx/emqx
 EMQX_TAG=build-alpine-amd64
diff --git a/Makefile b/Makefile
index 7ccb91c3c..f9310e636 100644
--- a/Makefile
+++ b/Makefile
@@ -61,7 +61,7 @@ mix-deps-get: $(ELIXIR_COMMON_DEPS)
 	@mix deps.get
 
 .PHONY: eunit
-eunit: $(REBAR) conf-segs
+eunit: $(REBAR) merge-config
 	@ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(PROFILE)-eunit
 
 .PHONY: proper
@@ -69,11 +69,11 @@ proper: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) proper -d test/props -c
 
 .PHONY: test-compile
-test-compile: $(REBAR) conf-segs
+test-compile: $(REBAR) merge-config
 	$(REBAR) as test compile
 
 .PHONY: ct
-ct: $(REBAR) conf-segs
+ct: $(REBAR) merge-config
 	@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-ct
 
 .PHONY: static_checks
@@ -224,7 +224,7 @@ ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir)
 $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt))))
 
 .PHONY:
-conf-segs:
+merge-config:
 	@$(SCRIPTS)/merge-config.escript
 	@$(SCRIPTS)/merge-i18n.escript
 
diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index b9ab6a8bf..4dd74989a 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -28,7 +28,6 @@
     boot_modules/1,
     start_apps/1,
     start_apps/2,
-    start_app/4,
     stop_apps/1,
     reload/2,
     app_path/2,
@@ -165,13 +164,15 @@ start_apps(Apps) ->
     start_apps(Apps, fun(_) -> ok end).
 
 -spec start_apps(Apps :: apps(), Handler :: special_config_handler()) -> ok.
-start_apps(Apps, Handler) when is_function(Handler) ->
+start_apps(Apps, SpecAppConfig) when is_function(SpecAppConfig) ->
     %% Load all application code to beam vm first
     %% Because, minirest, ekka etc.. application will scan these modules
     lists:foreach(fun load/1, [emqx | Apps]),
+    %% load emqx_conf config before starting ekka
+    render_and_load_app_config(emqx_conf),
     ok = start_ekka(),
     ok = emqx_ratelimiter_SUITE:load_conf(),
-    lists:foreach(fun(App) -> start_app(App, Handler) end, [emqx | Apps]).
+    lists:foreach(fun(App) -> start_app(App, SpecAppConfig) end, [emqx | Apps]).
 
 load(App) ->
     case application:load(App) of
@@ -180,13 +181,35 @@ load(App) ->
         {error, Reason} -> error({failed_to_load_app, App, Reason})
     end.
 
-start_app(App, Handler) ->
-    start_app(
-        App,
-        app_schema(App),
-        app_path(App, filename:join(["etc", app_conf_file(App)])),
-        Handler
-    ).
+render_and_load_app_config(App) ->
+    Schema = app_schema(App),
+    Conf = app_path(App, filename:join(["etc", app_conf_file(App)])),
+    try
+        do_render_app_config(App, Schema, Conf)
+    catch
+        throw:E:St ->
+            %% turn throw into error
+            error({Conf, E, St})
+    end.
+
+do_render_app_config(App, Schema, ConfigFile) ->
+    Vars = mustache_vars(App),
+    RenderedConfigFile = render_config_file(ConfigFile, Vars),
+    read_schema_configs(Schema, RenderedConfigFile),
+    force_set_config_file_paths(App, [RenderedConfigFile]),
+    copy_certs(App, RenderedConfigFile),
+    ok.
+
+start_app(App, SpecAppConfig) ->
+    render_and_load_app_config(App),
+    SpecAppConfig(App),
+    case application:ensure_all_started(App) of
+        {ok, _} ->
+            ok = ensure_dashboard_listeners_started(App),
+            ok;
+        {error, Reason} ->
+            error({failed_to_start_app, App, Reason})
+    end.
 
 app_conf_file(emqx_conf) -> "emqx.conf.all";
 app_conf_file(App) -> atom_to_list(App) ++ ".conf".
@@ -208,21 +231,6 @@ mustache_vars(App) ->
         {platform_log_dir, app_path(App, "log")}
     ].
 
-start_app(App, Schema, ConfigFile, SpecAppConfig) ->
-    Vars = mustache_vars(App),
-    RenderedConfigFile = render_config_file(ConfigFile, Vars),
-    read_schema_configs(Schema, RenderedConfigFile),
-    force_set_config_file_paths(App, [RenderedConfigFile]),
-    copy_certs(App, RenderedConfigFile),
-    SpecAppConfig(App),
-    case application:ensure_all_started(App) of
-        {ok, _} ->
-            ok = ensure_dashboard_listeners_started(App),
-            ok;
-        {error, Reason} ->
-            error({failed_to_start_app, App, Reason})
-    end.
-
 render_config_file(ConfigFile, Vars0) ->
     Temp =
         case file:read_file(ConfigFile) of
diff --git a/mix.exs b/mix.exs
index a3c18842c..f43ca7119 100644
--- a/mix.exs
+++ b/mix.exs
@@ -385,8 +385,8 @@ defmodule EMQXUmbrella.MixProject do
 
     assigns = template_vars(release, release_type, package_type, edition_type)
 
-    # This is generated by `scripts/merge-config.escript` or `make
-    # conf-segs`.  So, this should be run before the release.
+    # This is generated by `scripts/merge-config.escript` or `make merge-config`
+    # So, this should be run before the release.
     # TODO: run as a "compiler" step???
     render_template(
       "apps/emqx_conf/etc/emqx.conf.all",
diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript
index f78083ee1..1b30dbd1d 100755
--- a/scripts/merge-config.escript
+++ b/scripts/merge-config.escript
@@ -29,7 +29,7 @@ main(_) ->
 
     case IsEnterprise of
         true ->
-            EnterpriseCfgs = get_all_cfgs("lib-ee/"),
+            EnterpriseCfgs = get_all_cfgs("lib-ee"),
             EnterpriseConf = merge("", EnterpriseCfgs),
             ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf);
         false ->

From c9d84b2fda8f32df7b3cdb166359df654aea8d00 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 7 Dec 2022 16:49:30 +0100
Subject: [PATCH 229/232] test: do not load emqx_conf app when it's not found

also remove stale code in emqx_common_test_helpers
---
 apps/emqx/test/emqx_common_test_helpers.erl | 43 ++++-----------------
 1 file changed, 7 insertions(+), 36 deletions(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 4dd74989a..f468ad283 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -16,7 +16,6 @@
 
 -module(emqx_common_test_helpers).
 
--define(THIS_APP, ?MODULE).
 -include_lib("common_test/include/ct.hrl").
 
 -type special_config_handler() :: fun().
@@ -169,7 +168,13 @@ start_apps(Apps, SpecAppConfig) when is_function(SpecAppConfig) ->
     %% Because, minirest, ekka etc.. application will scan these modules
     lists:foreach(fun load/1, [emqx | Apps]),
     %% load emqx_conf config before starting ekka
-    render_and_load_app_config(emqx_conf),
+    case application:load(emqx_conf) of
+        {error, _} ->
+            %% running test only for emqx app (standalone)
+            ok;
+        _ ->
+            render_and_load_app_config(emqx_conf)
+    end,
     ok = start_ekka(),
     ok = emqx_ratelimiter_SUITE:load_conf(),
     lists:foreach(fun(App) -> start_app(App, SpecAppConfig) end, [emqx | Apps]).
@@ -275,43 +280,9 @@ proj_root() ->
 deps_path(App, RelativePath) -> app_path(App, RelativePath).
 
 app_path(App, RelativePath) ->
-    ok = ensure_app_loaded(App),
     Lib = code:lib_dir(App),
     safe_relative_path(filename:join([Lib, RelativePath])).
 
-assert_app_loaded(App) ->
-    case code:lib_dir(App) of
-        {error, bad_name} -> error({not_loaded, ?THIS_APP});
-        _ -> ok
-    end.
-
-ensure_app_loaded(?THIS_APP) ->
-    ok = assert_app_loaded(?THIS_APP);
-ensure_app_loaded(App) ->
-    case code:lib_dir(App) of
-        {error, bad_name} ->
-            ok = assert_app_loaded(?THIS_APP),
-            Dir0 = code:lib_dir(?THIS_APP),
-            LibRoot = upper_level(Dir0),
-            Dir = filename:join([LibRoot, atom_to_list(App), "ebin"]),
-            case code:add_pathz(Dir) of
-                true -> ok;
-                {error, bad_directory} -> error({bad_directory, Dir})
-            end,
-            case application:load(App) of
-                ok -> ok;
-                {error, Reason} -> error({failed_to_load, App, Reason})
-            end,
-            ok = assert_app_loaded(App);
-        _ ->
-            ok
-    end.
-
-upper_level(Dir) ->
-    Split = filename:split(Dir),
-    UpperReverse = tl(lists:reverse(Split)),
-    filename:join(lists:reverse(UpperReverse)).
-
 safe_relative_path(Path) ->
     case filename:split(Path) of
         ["/" | T] ->

From 1c8a9079b9d6e4cd116bbbb08670fb5c52dd18b4 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 7 Dec 2022 17:27:59 +0100
Subject: [PATCH 230/232] ci: remove undefined matrix from artifact name

---
 .github/workflows/run_test_cases.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index deca9d3c9..6b94dbb95 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -167,7 +167,7 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs-${{ matrix.profile }}-${{ matrix.app[0] }}-${{ matrix.app[1] }}
+            name: logs-${{ matrix.app[0] }}-${{ matrix.app[1] }}
             path: source/_build/test/logs
 
     ct:
@@ -209,7 +209,7 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs-${{ matrix.profile }}-${{ matrix.app[0] }}-${{ matrix.app[1] }}
+            name: logs-${{ matrix.app[0] }}-${{ matrix.app[1] }}
             path: source/_build/test/logs
 
     make_cover:

From c87c9e886ea10a704bac5397cff136bba5bbeb65 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 7 Dec 2022 17:34:17 +0100
Subject: [PATCH 231/232] ci: rename build step for docker-ct

---
 .github/workflows/run_test_cases.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 6b94dbb95..896617d15 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -146,7 +146,7 @@ jobs:
             path: .
         - name: unzip source code
           run: unzip -q source.zip
-        - name: docker compose up
+        - name: run tests
           working-directory: source
           env:
             MONGO_TAG: 5

From dc14cd450dcd5d888eaecb30d8a4ecef3784c887 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 7 Dec 2022 18:27:06 +0100
Subject: [PATCH 232/232] test: render config for emqx_conf only for ee bridge
 tests

---
 apps/emqx/test/emqx_common_test_helpers.erl     | 12 +++---------
 .../emqx_bridge_impl_kafka_producer_SUITE.erl   |  2 ++
 .../test/emqx_ee_bridge_influxdb_SUITE.erl      | 14 ++++++++++----
 .../test/emqx_ee_bridge_mongodb_SUITE.erl       | 17 +++++++++++------
 4 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index f468ad283..87d9c1368 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -33,7 +33,8 @@
     proj_root/0,
     deps_path/2,
     flush/0,
-    flush/1
+    flush/1,
+    render_and_load_app_config/1
 ]).
 
 -export([
@@ -167,14 +168,6 @@ start_apps(Apps, SpecAppConfig) when is_function(SpecAppConfig) ->
     %% Load all application code to beam vm first
     %% Because, minirest, ekka etc.. application will scan these modules
     lists:foreach(fun load/1, [emqx | Apps]),
-    %% load emqx_conf config before starting ekka
-    case application:load(emqx_conf) of
-        {error, _} ->
-            %% running test only for emqx app (standalone)
-            ok;
-        _ ->
-            render_and_load_app_config(emqx_conf)
-    end,
     ok = start_ekka(),
     ok = emqx_ratelimiter_SUITE:load_conf(),
     lists:foreach(fun(App) -> start_app(App, SpecAppConfig) end, [emqx | Apps]).
@@ -187,6 +180,7 @@ load(App) ->
     end.
 
 render_and_load_app_config(App) ->
+    load(App),
     Schema = app_schema(App),
     Conf = app_path(App, filename:join(["etc", app_conf_file(App)])),
     try
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 2eef1170d..0a26e5d26 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -67,6 +67,8 @@ init_per_suite(Config) ->
     %% Need to unload emqx_authz. See emqx_machine_SUITE:init_per_suite for
     %% more info.
     application:unload(emqx_authz),
+    %% some configs in emqx_conf app are mandatory
+    emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
     emqx_common_test_helpers:start_apps(
         [emqx_conf, emqx_rule_engine, emqx_bridge, emqx_management, emqx_dashboard],
         fun set_special_configs/1
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
index d22c6abc0..c2ac45551 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
@@ -89,8 +89,7 @@ init_per_group(InfluxDBType, Config0) when
             ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
             ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
             emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge]),
+            ok = start_apps(),
             {ok, _} = application:ensure_all_started(emqx_connector),
             Config = [{use_tls, UseTLS} | Config0],
             {Name, ConfigString, InfluxDBConfig} = influxdb_config(
@@ -158,8 +157,7 @@ init_per_group(InfluxDBType, Config0) when
             ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
             ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
             emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge]),
+            ok = start_apps(),
             {ok, _} = application:ensure_all_started(emqx_connector),
             Config = [{use_tls, UseTLS} | Config0],
             {Name, ConfigString, InfluxDBConfig} = influxdb_config(
@@ -855,3 +853,11 @@ t_write_failure(Config) ->
         end
     ),
     ok.
+
+start_apps() ->
+    %% some configs in emqx_conf app are mandatory
+    %% we want to make sure they are loaded before
+    %% ekka start in emqx_common_test_helpers:start_apps/1
+    emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
+    ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+    ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge]).
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
index bc8c0b04f..35698f812 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
@@ -40,8 +40,7 @@ init_per_group(Type = rs, Config) ->
     MongoPort = list_to_integer(os:getenv("MONGO_RS_PORT", "27017")),
     case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
         true ->
-            ensure_loaded(),
-            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            ok = start_apps(),
             emqx_mgmt_api_test_util:init_suite(),
             {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type),
             [
@@ -60,8 +59,7 @@ init_per_group(Type = sharded, Config) ->
     MongoPort = list_to_integer(os:getenv("MONGO_SHARDED_PORT", "27017")),
     case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
         true ->
-            ensure_loaded(),
-            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            ok = start_apps(),
             emqx_mgmt_api_test_util:init_suite(),
             {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type),
             [
@@ -80,8 +78,7 @@ init_per_group(Type = single, Config) ->
     MongoPort = list_to_integer(os:getenv("MONGO_SINGLE_PORT", "27017")),
     case emqx_common_test_helpers:is_tcp_server_available(MongoHost, MongoPort) of
         true ->
-            ensure_loaded(),
-            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            ok = start_apps(),
             emqx_mgmt_api_test_util:init_suite(),
             {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type),
             [
@@ -121,6 +118,14 @@ end_per_testcase(_Testcase, Config) ->
 %% Helper fns
 %%------------------------------------------------------------------------------
 
+start_apps() ->
+    ensure_loaded(),
+    %% some configs in emqx_conf app are mandatory,
+    %% we want to make sure they are loaded before
+    %% ekka start in emqx_common_test_helpers:start_apps/1
+    emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
+    ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]).
+
 ensure_loaded() ->
     _ = application:load(emqx_ee_bridge),
     _ = emqx_ee_bridge:module_info(),