From e4eba157c332b445bff82726f66deecb495942b2 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 25 Jul 2022 09:36:45 +0800 Subject: [PATCH 01/71] feat: add emqx enterprise app connector & bridge --- lib-ee/emqx_ee_bridge/.gitignore | 19 +++++++++++++ lib-ee/emqx_ee_bridge/README.md | 9 +++++++ lib-ee/emqx_ee_bridge/rebar.config | 6 +++++ .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 14 ++++++++++ .../emqx_ee_bridge/src/emqx_ee_bridge_app.erl | 17 ++++++++++++ .../emqx_ee_bridge/src/emqx_ee_bridge_sup.erl | 27 +++++++++++++++++++ lib-ee/emqx_ee_connector/.gitignore | 19 +++++++++++++ lib-ee/emqx_ee_connector/README.md | 9 +++++++ lib-ee/emqx_ee_connector/rebar.config | 6 +++++ .../src/emqx_ee_connector.app.src | 14 ++++++++++ .../src/emqx_ee_connector_app.erl | 17 ++++++++++++ .../src/emqx_ee_connector_sup.erl | 27 +++++++++++++++++++ .../src/emqx_enterprise_conf.app.src | 1 - rebar.config.erl | 4 ++- 14 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 lib-ee/emqx_ee_bridge/.gitignore create mode 100644 lib-ee/emqx_ee_bridge/README.md create mode 100644 lib-ee/emqx_ee_bridge/rebar.config create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl create mode 100644 lib-ee/emqx_ee_connector/.gitignore create mode 100644 lib-ee/emqx_ee_connector/README.md create mode 100644 lib-ee/emqx_ee_connector/rebar.config create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl diff --git a/lib-ee/emqx_ee_bridge/.gitignore b/lib-ee/emqx_ee_bridge/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/lib-ee/emqx_ee_bridge/README.md b/lib-ee/emqx_ee_bridge/README.md new file mode 100644 index 000000000..5cb4d8694 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/README.md @@ -0,0 +1,9 @@ +emqx_ee_bridge +===== + +An OTP application + +Build +----- + + $ rebar3 compile diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config new file mode 100644 index 000000000..5dd22ccef --- /dev/null +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -0,0 +1,6 @@ +{erl_opts, [debug_info]}. +{deps, []}. + +{shell, [ + {apps, [emqx_ee_bridge]} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src new file mode 100644 index 000000000..58846577e --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -0,0 +1,14 @@ +{application, emqx_ee_bridge, [ + {description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_ee_bridge_app, []}}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl new file mode 100644 index 000000000..22268e285 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl @@ -0,0 +1,17 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_ee_bridge_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl new file mode 100644 index 000000000..5a2484442 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl @@ -0,0 +1,27 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{ + strategy => one_for_all, + intensity => 0, + period => 1 + }, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/lib-ee/emqx_ee_connector/.gitignore b/lib-ee/emqx_ee_connector/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/lib-ee/emqx_ee_connector/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/lib-ee/emqx_ee_connector/README.md b/lib-ee/emqx_ee_connector/README.md new file mode 100644 index 000000000..e665af458 --- /dev/null +++ b/lib-ee/emqx_ee_connector/README.md @@ -0,0 +1,9 @@ +emqx_ee_connector +===== + +An OTP application + +Build +----- + + $ rebar3 compile diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config new file mode 100644 index 000000000..cba9da867 --- /dev/null +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -0,0 +1,6 @@ +{erl_opts, [debug_info]}. +{deps, []}. + +{shell, [ + {apps, [emqx_ee_connector]} +]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src new file mode 100644 index 000000000..87f68afb6 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -0,0 +1,14 @@ +{application, emqx_ee_connector, [ + {description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_ee_connector_app, []}}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl new file mode 100644 index 000000000..13db2ccb0 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl @@ -0,0 +1,17 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_ee_connector_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl new file mode 100644 index 000000000..f8a219fdc --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl @@ -0,0 +1,27 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{ + strategy => one_for_all, + intensity => 0, + period => 1 + }, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src b/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src index 37cb78b54..a25ce1aae 100644 --- a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src +++ b/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src @@ -9,6 +9,5 @@ {env, []}, {modules, []}, - {licenses, ["Apache 2.0"]}, {links, []} ]}. diff --git a/rebar.config.erl b/rebar.config.erl index 6748f7ce8..96754f7f8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -378,7 +378,9 @@ is_app(Name) -> relx_apps_per_edition(ee) -> [ emqx_license, - {emqx_enterprise_conf, load} + {emqx_enterprise_conf, load}, + emqx_ee_connector, + emqx_ee_bridge ]; relx_apps_per_edition(ce) -> []. From f00a7417bb6eb7c0754cfa43e29919d938e29014 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 25 Jul 2022 11:18:38 +0800 Subject: [PATCH 02/71] feat: hstream bridge & connector --- apps/emqx_bridge/src/emqx_bridge_schema.erl | 14 +- .../i18n/emqx_ee_bridge_hstream.conf | 84 +++++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 15 ++ .../src/emqx_ee_bridge_hstream.erl | 57 +++++ .../i18n/emqx_ee_connector_hstream.conf | 43 ++++ lib-ee/emqx_ee_connector/rebar.config | 4 +- .../src/emqx_ee_connector_hstream.erl | 218 ++++++++++++++++++ 7 files changed, 430 insertions(+), 5 deletions(-) create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index d512df323..d162f0c13 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -45,10 +45,16 @@ http_schema(Method) -> end, ?CONN_TYPES ), - hoconsc:union([ - ref(emqx_bridge_webhook_schema, Method) - | Schemas - ]). + ExtSchemas = [ref(Module, Method) || Module <- schema_modules()], + hoconsc:union(Schemas ++ ExtSchemas). + +-ifdef(EMQX_RELEASE_EDITION). +schema_modules() -> + [emqx_bridge_webhook_schema] ++ emqx_ee_bridge:schema_modules(). +-else. +schema_modules() -> + [emqx_bridge_webhook_schema]. +-endif. common_bridge_fields(ConnectorRef) -> [ diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf new file mode 100644 index 000000000..a110dbf27 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf @@ -0,0 +1,84 @@ +emqx_ee_bridge_hstream { + local_topic { + desc { + en: """ +The MQTT topic filter to be forwarded to the HStreamDB. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded. +""" + zh: """ +发送到 'local_topic' 的消息都会转发到 HStreamDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。 +""" + } + label { + en: "Local Topic" + zh: "本地 Topic" + } + } + payload { + desc { + en: """The payload to be forwarded to the HStreamDB. Placeholders supported.""" + zh: """要转发到 HStreamDB 的数据内容，支持占位符""" + } + label { + en: "Payload" + zh: "消息内容" + } + } + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用 Bridge""" + } + label { + en: "Enable Or Disable Bridge" + zh: "启用/禁用 Bridge" + } + } + config_direction { + desc { + en: """The direction of this bridge, MUST be 'egress'""" + zh: """Bridge 的方向， 必须是 egress""" + } + label { + en: "Bridge Direction" + zh: "Bridge 方向" + } + } + + desc_config { + desc { + en: """Configuration for an HStreamDB bridge.""" + zh: """HStreamDB Bridge 配置""" + } + label: { + en: "HStreamDB Bridge Configuration" + zh: "HStreamDB Bridge 配置" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: "Bridge Type" + zh: "Bridge 类型" + } + } + + desc_name { + desc { + en: """Bridge name, used as a human-readable description of the bridge.""" + zh: """Bridge 名字，Bridge 的可读描述""" + } + label { + en: "Bridge Name" + zh: "Bridge 名字" + } + } +} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl new file mode 100644 index 000000000..d18fa1fde --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -0,0 +1,15 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge). + +-export([ + schema_modules/0, + info_example_basic/2 +]). + +schema_modules() -> + [emqx_ee_bridge_hstream]. + +info_example_basic(_Type, _Direction) -> + #{}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl new file mode 100644 index 000000000..6b01d2c0b --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_hstream). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, enum/1]). + +-export([ + roots/0, + fields/1, + desc/1 +]). + +%%====================================================================================== +%% Hocon Schema Definitions +% namespace() -> "bridge". + +roots() -> []. + +fields("config") -> + ExtConfig = [ + {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, + {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})} + ], + basic_config() ++ ExtConfig; +fields("post") -> + [type_field(), name_field() | fields("config")]; +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:metrics_status_fields() ++ fields("post"). + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for HStream using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +basic_config() -> + Basic = [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})} + ], + emqx_ee_connector_hstream:fields(config) ++ Basic. + +%%====================================================================================== + +type_field() -> + % {type, mk(hstream, #{required => true, desc => ?DESC("desc_type")})}. + {type, mk(hstream, #{required => true, desc => <<"desc_type">>})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf new file mode 100644 index 000000000..d99b09c5f --- /dev/null +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf @@ -0,0 +1,43 @@ + +emqx_ee_connector_hstream { + url { + desc { + en: """Hstream Server URL""" + zh: """HStream 服务器 URL""" + } + label { + en: """Hstream Server URL""" + zh: """HStream 服务器 URL""" + } + } + stream_name { + desc { + en: """Hstream Stream Name""" + zh: """HStream 流名称""" + } + label { + en: """Hstream Stream Name""" + zh: """HStream 流名称""" + } + } + ordering_key { + desc { + en: """Hstream Ordering Key""" + zh: """HStream 分区键""" + } + label { + en: """Hstream Ordering Key""" + zh: """HStream 分区键""" + } + } + pool_size { + desc { + en: """Hstream Pool Size""" + zh: """HStream 连接池大小""" + } + label { + en: """Hstream Pool Size""" + zh: """HStream 连接池大小""" + } + } +} diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index cba9da867..1f16e648e 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,5 +1,7 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.4"}}} +]}. {shell, [ {apps, [emqx_ee_connector]} diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl new file mode 100644 index 000000000..ea3a73035 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl @@ -0,0 +1,218 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_connector_hstream). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-import(hoconsc, [mk/2, enum/1]). + +-behaviour(emqx_resource). + +%% callbacks of behaviour emqx_resource +-export([ + on_start/2, + on_stop/2, + on_query/4, + on_get_status/2 +]). + +-export([ + on_flush_result/1 +]). + +-export([ + roots/0, + fields/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% resource callback + +on_start(InstId, Config) -> + start_client(InstId, Config). + +on_stop(InstId, #{client := Client, producer := Producer}) -> + StopClientRes = hstreamdb:stop_client(Client), + StopProducerRes = hstreamdb:stop_producer(Producer), + ?SLOG(info, #{ + msg => "stop hstream connector", + connector => InstId, + client => Client, + producer => Producer, + stop_client => StopClientRes, + stop_producer => StopProducerRes + }). + +on_query(_InstId, {OrderingKey, Payload, Record}, AfterQuery, #{producer := Producer}) -> + Record = hstreamdb:to_record(OrderingKey, raw, Payload), + do_append(AfterQuery, false, Producer, Record). + +on_get_status(_InstId, #{client := Client}) -> + is_alive(Client). + +%% ------------------------------------------------------------------------------------------------- +%% hstream batch callback +%% TODO: maybe remove it after disk cache is ready + +on_flush_result({{flush, _Stream, _Records}, {ok, _Resp}}) -> + ok; +on_flush_result({{flush, _Stream, _Records}, {error, _Reason}}) -> + ok. + +%% ------------------------------------------------------------------------------------------------- +%% schema + +roots() -> + fields(config). + +fields(config) -> + [ + {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, + {stream, mk(binary(), #{required => true, desc => ?DESC("stream")})}, + {ordering_key, mk(binary(), #{required => true, desc => ?DESC("ordering_key")})}, + {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})} + ]. + +%% ------------------------------------------------------------------------------------------------- +%% internal functions + +start_client(InstId, Config = #{server := Server, pool_size := PoolSize}) -> + ?SLOG(info, #{ + msg => "starting hstream connector: client", + connector => InstId, + config => Config + }), + ClientName = client_name(InstId), + ClientOptions = [ + {url, Server}, + {rpc_options, #{pool_size => PoolSize}} + ], + case hstreamdb:start_client(ClientName, ClientOptions) of + {ok, Client} -> + case is_alive(Client) of + true -> + ?SLOG(info, #{ + msg => "hstream connector: client started", + connector => InstId, + client => Client + }), + start_producer(InstId, Client, Config); + _ -> + ?SLOG(error, #{ + msg => "hstream connector: client not alive", + connector => InstId + }), + {error, connect_failed} + end; + {error, {already_started, Pid}} -> + ?SLOG(info, #{ + msg => "starting hstream connector: client, find old client. restart client", + old_client_pid => Pid, + old_client_name => ClientName + }), + _ = hstreamdb:stop_client(ClientName), + start_client(InstId, Config); + {error, Error} -> + ?SLOG(error, #{ + msg => "hstream connector: client failed", + connector => InstId, + reason => Error + }), + {error, Error} + end. + +is_alive(Client) -> + case hstreamdb:echo(Client) of + {ok, _Echo} -> + true; + _ErrorEcho -> + false + end. + +start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSize}) -> + %% TODO: change these batch options after we have better disk cache. + BatchSize = maps:get(batch_size, Options, 100), + Interval = maps:get(batch_interval, Options, 1000), + ProducerOptions = [ + {stream, Stream}, + {callback, {?MODULE, on_flush_result, []}}, + {max_records, BatchSize}, + {interval, Interval}, + {pool_size, PoolSize} + ], + Name = produce_name(InstId), + ?SLOG(info, #{ + msg => "starting hstream connector: producer", + connector => InstId + }), + case hstreamdb:start_producer(Client, Name, ProducerOptions) of + {ok, Producer} -> + ?SLOG(info, #{ + msg => "hstream connector: producer started" + }), + EnableBatch = maps:get(enable_batch, Options, false), + #{client => Client, producer => Producer, enable_batch => EnableBatch}; + {error, {already_started, Pid}} -> + ?SLOG(info, #{ + msg => "starting hstream connector: producer, find old producer. restart producer", + old_producer_pid => Pid, + old_producer_name => Name + }), + _ = hstreamdb:stop_producer(Name), + start_producer(InstId, Client, Options); + {error, Reason} -> + ?SLOG(error, #{ + msg => "starting hstream connector: producer, failed", + reason => Reason + }), + {error, Reason} + end. + +%% TODO: this append is async, remove or change it after we have better disk cache. +% do_append(AfterQuery, true, Producer, Record) -> +% case hstreamdb:append(Producer, Record) of +% ok -> +% ?SLOG(debug, #{ +% msg => "hstream producer async append success", +% record => Record +% }), +% emqx_resource:query_success(AfterQuery); +% {error, Reason} -> +% ?SLOG(error, #{ +% msg => "hstream producer async append failed", +% reason => Reason, +% record => Record +% }), +% emqx_resource:query_failed(AfterQuery) +% end; +do_append(AfterQuery, false, Producer, Record) -> + %% TODO: this append is sync, but it does not support [Record], can only append one Record. + %% Change it after we have better dick cache. + case hstreamdb:append_flush(Producer, Record) of + {ok, _} -> + ?SLOG(debug, #{ + msg => "hstream producer sync append success", + record => Record + }), + emqx_resource:query_success(AfterQuery); + {error, Reason} -> + ?SLOG(error, #{ + msg => "hstream producer sync append failed", + reason => Reason, + record => Record + }), + emqx_resource:query_failed(AfterQuery) + end. + +client_name(InstId) -> + "backend_hstream_client:" ++ to_string(InstId). + +produce_name(ActionId) -> + list_to_atom("backend_hstream_producer:" ++ to_string(ActionId)). + +to_string(List) when is_list(List) -> List; +to_string(Bin) when is_binary(Bin) -> binary_to_list(Bin); +to_string(Atom) when is_atom(Atom) -> atom_to_list(Atom). From 98b36c468167135300e069989b1553db8fc40331 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 25 Jul 2022 19:03:53 +0800 Subject: [PATCH 03/71] fix: hstream db connector , TODO: start apps --- apps/emqx_bridge/src/emqx_bridge_api.erl | 16 +++++-- apps/emqx_bridge/src/emqx_bridge_resource.erl | 24 ++++++++-- apps/emqx_bridge/src/emqx_bridge_schema.erl | 10 +++- .../src/emqx_resource_manager.erl | 2 +- .../emqx_ee_bridge/include/emqx_ee_bridge.hrl | 18 ++++++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 31 +++++++++++-- .../src/emqx_ee_bridge_hstream.erl | 46 ++++++++++++++++--- .../i18n/emqx_ee_connector_hstream.conf | 2 +- .../src/emqx_ee_connector.app.src | 3 +- .../src/emqx_ee_connector_hstream.erl | 23 ++++++++-- scripts/merge-i18n.escript | 2 +- 11 files changed, 151 insertions(+), 26 deletions(-) create mode 100644 lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index bc9b6c5a2..0d3facc5c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -163,7 +163,7 @@ bridge_info_examples(Method) -> }). conn_bridge_examples(Method) -> - lists:foldl( + Fun = fun(Type, Acc) -> SType = atom_to_list(Type), KeyIngress = bin(SType ++ "_ingress"), @@ -179,9 +179,17 @@ conn_bridge_examples(Method) -> } }) end, - #{}, - ?CONN_TYPES - ). + Broker = lists:foldl(Fun, #{}, ?CONN_TYPES), + EE = ee_conn_bridge_examples(Method), + maps:merge(Broker, EE). + +-ifdef(EMQX_RELEASE_EDITION). +ee_conn_bridge_examples(Method) -> + emqx_ee_bridge:conn_bridge_examples(Method). +-else. +ee_conn_bridge_examples(_Method) -> + #{}. +-endif. info_example(Type, Direction, Method) -> maps:merge( diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 678aa1f10..c82a290b3 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -42,10 +42,18 @@ reset_metrics/1 ]). +-ifdef(EMQX_RELEASE_EDITION). +bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; +bridge_to_resource_type(mqtt) -> emqx_connector_mqtt; +bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http; +bridge_to_resource_type(webhook) -> emqx_connector_http; +bridge_to_resource_type(BridgeType) -> emqx_ee_bridge:resource_type(BridgeType). +-else. bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; bridge_to_resource_type(mqtt) -> emqx_connector_mqtt; bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http; bridge_to_resource_type(webhook) -> emqx_connector_http. +-endif. resource_id(BridgeId) when is_binary(BridgeId) -> <<"bridge:", BridgeId/binary>>. @@ -254,7 +262,7 @@ parse_confs( request_timeout => ReqTimeout } }; -parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) when +parse_confs(Type = mqtt, Name, #{connector := ConnId, direction := Direction} = Conf) when is_binary(ConnId) -> case emqx_connector:parse_connector_id(ConnId) of @@ -270,7 +278,7 @@ parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) w {_ConnType, _ConnName} -> error({cannot_use_connector_with_different_type, ConnId}) end; -parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when +parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when is_map(ConnectorConfs) -> make_resource_confs( @@ -279,7 +287,17 @@ parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = maps:without([connector, direction], Conf), Type, Name - ). + ); +parse_confs(Type, Name, Conf) -> + parse_enterprise_confs(Type, Name, Conf). + +-ifdef(EMQX_RELEASE_EDITION). +parse_enterprise_confs(Type, Name, Conf) -> + emqx_ee_bridge:parse_conf(Type, Name, Conf). +-else. +parse_enterprise_confs(Type, Name, Conf) -> + error({not_supported, Type, Name}). +-endif. make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) -> BName = bridge_id(Type, Name), diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index d162f0c13..de27cd34c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -133,7 +133,7 @@ fields(bridges) -> #{desc => ?DESC("bridges_name")} )} || T <- ?CONN_TYPES - ]; + ] ++ ee_fields_bridges(); fields("metrics") -> [ {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})}, @@ -158,6 +158,14 @@ fields("node_status") -> {"status", mk(status(), #{})} ]. +-ifdef(EMQX_RELEASE_EDITION). +ee_fields_bridges() -> + emqx_ee_bridge:fields(bridges). +-else. +ee_fields_bridges() -> + []. +-endif. + desc(bridges) -> ?DESC("desc_bridges"); desc("metrics") -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 3a1afd27c..82be9c58f 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -214,7 +214,7 @@ get_metrics(ResId) -> reset_metrics(ResId) -> emqx_metrics_worker:reset_metrics(resource_metrics, ResId). -%% @doc Returns the data for all resorces +%% @doc Returns the data for all resources -spec list_all() -> [resource_data()] | []. list_all() -> try diff --git a/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl b/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl new file mode 100644 index 000000000..0065db56b --- /dev/null +++ b/lib-ee/emqx_ee_bridge/include/emqx_ee_bridge.hrl @@ -0,0 +1,18 @@ +-define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{ + matched => MATCH, + success => SUCC, + failed => FAILED, + rate => RATE, + rate_last5m => RATE_5, + rate_max => RATE_MAX +}). + +-define(METRICS_EXAMPLE, #{ + metrics => ?METRICS(0, 0, 0, 0, 0, 0), + node_metrics => [ + #{ + node => node(), + metrics => ?METRICS(0, 0, 0, 0, 0, 0) + } + ] +}). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index d18fa1fde..a94089c21 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -3,13 +3,38 @@ %%-------------------------------------------------------------------- -module(emqx_ee_bridge). +-import(hoconsc, [mk/2, enum/1, ref/2]). + -export([ schema_modules/0, - info_example_basic/2 + conn_bridge_examples/1, + resource_type/1, + parse_conf/3, + fields/1 ]). schema_modules() -> [emqx_ee_bridge_hstream]. -info_example_basic(_Type, _Direction) -> - #{}. +conn_bridge_examples(Method) -> + Fun = + fun(Module, Examples) -> + Example = erlang:apply(Module, conn_bridge_example, [Method]), + maps:merge(Examples, Example) + end, + lists:foldl(Fun, #{}, schema_modules()). + +resource_type(hstream) -> emqx_ee_connector_hstream; +resource_type(<<"hstream">>) -> emqx_ee_connector_hstream. + +parse_conf(_Type, _Name, Conf) -> + Conf. + +fields(bridges) -> + [ + {hstream, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), + #{desc => <<"hstream_webhook">>} + )} + ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl index 6b01d2c0b..42dd86f14 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl @@ -5,18 +5,53 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_ee_bridge.hrl"). -import(hoconsc, [mk/2, enum/1]). -export([ + conn_bridge_example/1 +]). + +-export([ + namespace/0, roots/0, fields/1, desc/1 ]). -%%====================================================================================== +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_example(Method) -> + #{ + <<"hstream">> => #{ + summary => <<"HStreamDB Bridge">>, + value => values(Method) + } + }. + +values(get) -> + maps:merge(values(post), ?METRICS_EXAMPLE); +values(post) -> + #{ + type => hstream, + name => <<"hstream_bridge_demo">>, + url => <<"http://127.0.0.1:6570">>, + stream => <<"stream1">>, + ordering_key => <<"${topic}">>, + pool_size => 8, + enable => true, + direction => egress, + local_topic => <<"local/topic/#">>, + payload => <<"${payload}">> + }; +values(put) -> + values(post). + +%% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions -% namespace() -> "bridge". +namespace() -> "bridge". roots() -> []. @@ -47,11 +82,10 @@ basic_config() -> ], emqx_ee_connector_hstream:fields(config) ++ Basic. -%%====================================================================================== - +%% ------------------------------------------------------------------------------------------------- +%% internal type_field() -> - % {type, mk(hstream, #{required => true, desc => ?DESC("desc_type")})}. - {type, mk(hstream, #{required => true, desc => <<"desc_type">>})}. + {type, mk(enum([hstream]), #{required => true, desc => ?DESC("desc_type")})}. name_field() -> {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf index d99b09c5f..605944997 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf @@ -38,6 +38,6 @@ emqx_ee_connector_hstream { label { en: """Hstream Pool Size""" zh: """HStream 连接池大小""" - } + } } } diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 87f68afb6..214488f80 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -5,7 +5,8 @@ {mod, {emqx_ee_connector_app, []}}, {applications, [ kernel, - stdlib + stdlib, + hstreamdb_erl ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl index ea3a73035..7ba92f809 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl @@ -51,7 +51,12 @@ on_query(_InstId, {OrderingKey, Payload, Record}, AfterQuery, #{producer := Prod do_append(AfterQuery, false, Producer, Record). on_get_status(_InstId, #{client := Client}) -> - is_alive(Client). + case is_alive(Client) of + true -> + connected; + false -> + disconnected + end. %% ------------------------------------------------------------------------------------------------- %% hstream batch callback @@ -71,15 +76,23 @@ roots() -> fields(config) -> [ {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, - {stream, mk(binary(), #{required => true, desc => ?DESC("stream")})}, + {stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})}, {ordering_key, mk(binary(), #{required => true, desc => ?DESC("ordering_key")})}, {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})} ]. %% ------------------------------------------------------------------------------------------------- %% internal functions +start_client(InstId, Config) -> + try + do_start_client(InstId, Config) + catch + E:R:S -> + io:format("E:R:S ~p:~p ~n~p~n", [E, R, S]), + error(E) + end. -start_client(InstId, Config = #{server := Server, pool_size := PoolSize}) -> +do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> ?SLOG(info, #{ msg => "starting hstream connector: client", connector => InstId, @@ -87,7 +100,7 @@ start_client(InstId, Config = #{server := Server, pool_size := PoolSize}) -> }), ClientName = client_name(InstId), ClientOptions = [ - {url, Server}, + {url, binary_to_list(Server)}, {rpc_options, #{pool_size => PoolSize}} ], case hstreamdb:start_client(ClientName, ClientOptions) of @@ -154,7 +167,7 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi msg => "hstream connector: producer started" }), EnableBatch = maps:get(enable_batch, Options, false), - #{client => Client, producer => Producer, enable_batch => EnableBatch}; + {ok, #{client => Client, producer => Producer, enable_batch => EnableBatch}}; {error, {already_started, Pid}} -> ?SLOG(info, #{ msg => "starting hstream connector: producer, find old producer. restart producer", diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript index 9f8ac91ff..493798738 100755 --- a/scripts/merge-i18n.escript +++ b/scripts/merge-i18n.escript @@ -4,7 +4,7 @@ main(_) -> BaseConf = <<"">>, - Cfgs = get_all_cfgs("apps/"), + Cfgs = get_all_cfgs("apps/") ++ get_all_cfgs("lib-ee/"), Conf = [merge(BaseConf, Cfgs), io_lib:nl() ], From fa4bc921ac0876622183258c28e2b868f775d8be Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 26 Jul 2022 15:28:56 +0800 Subject: [PATCH 04/71] fix: hstream db connector & bridge, TODO: SUITE --- apps/emqx_bridge/src/emqx_bridge_app.erl | 11 ++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 2 +- .../src/emqx_ee_bridge_hstream.erl | 4 +- .../src/emqx_ee_connector_hstream.erl | 50 ++++++++++++++++--- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 3fc4d57ba..3e67290d6 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -29,6 +29,7 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_bridge_sup:start_link(), + ok = start_ee_apps(), ok = emqx_bridge:load(), ok = emqx_bridge:load_hook(), ok = emqx_config_handler:add_handler(?LEAF_NODE_HDLR_PATH, ?MODULE), @@ -41,6 +42,16 @@ stop(_State) -> ok = emqx_bridge:unload_hook(), ok. +-ifdef(EMQX_RELEASE_EDITION). +start_ee_apps() -> + {ok, _} = application:ensure_all_started(emqx_ee_bridge), + {ok, _} = application:ensure_all_started(emqx_ee_connector), + ok. +-else. +start_ee_apps() -> + ok. +-endif. + %% NOTE: We depends on the `emqx_bridge:pre_config_update/3` to restart/stop the %% underlying resources. pre_config_update(_, {_Oper, _, _}, undefined) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index a94089c21..6b2987307 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -35,6 +35,6 @@ fields(bridges) -> {hstream, mk( hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), - #{desc => <<"hstream_webhook">>} + #{desc => <<"emqx enterprise config">>} )} ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl index 42dd86f14..56bac456e 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl @@ -36,10 +36,10 @@ values(get) -> values(post) -> #{ type => hstream, - name => <<"hstream_bridge_demo">>, + name => <<"demo">>, url => <<"http://127.0.0.1:6570">>, stream => <<"stream1">>, - ordering_key => <<"${topic}">>, + ordering_key => <<"some_key">>, pool_size => 8, enable => true, direction => egress, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl index 7ba92f809..fa8439851 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl @@ -46,9 +46,14 @@ on_stop(InstId, #{client := Client, producer := Producer}) -> stop_producer => StopProducerRes }). -on_query(_InstId, {OrderingKey, Payload, Record}, AfterQuery, #{producer := Producer}) -> - Record = hstreamdb:to_record(OrderingKey, raw, Payload), - do_append(AfterQuery, false, Producer, Record). +on_query( + _InstId, + {send_message, Data}, + AfterQuery, + #{producer := Producer, ordering_key := OrderingKey, payload := Payload} +) -> + Record = to_record(OrderingKey, Payload, Data), + do_append(AfterQuery, Producer, Record). on_get_status(_InstId, #{client := Client}) -> case is_alive(Client) of @@ -88,8 +93,13 @@ start_client(InstId, Config) -> do_start_client(InstId, Config) catch E:R:S -> - io:format("E:R:S ~p:~p ~n~p~n", [E, R, S]), - error(E) + ?SLOG(error, #{ + msg => "start hstream connector error", + connector => InstId, + error => E, + reason => R, + stack => S + }) end. do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> @@ -167,7 +177,18 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi msg => "hstream connector: producer started" }), EnableBatch = maps:get(enable_batch, Options, false), - {ok, #{client => Client, producer => Producer, enable_batch => EnableBatch}}; + PayloadBin = maps:get(payload, Options, <<"">>), + Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin), + OrderingKeyBin = maps:get(ordering_key, Options, <<"">>), + OrderingKey = emqx_plugin_libs_rule:preproc_tmpl(OrderingKeyBin), + State = #{ + client => Client, + producer => Producer, + enable_batch => EnableBatch, + ordering_key => OrderingKey, + payload => Payload + }, + {ok, State}; {error, {already_started, Pid}} -> ?SLOG(info, #{ msg => "starting hstream connector: producer, find old producer. restart producer", @@ -184,6 +205,19 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi {error, Reason} end. +to_record(OrderingKeyTmpl, PayloadTmpl, Data) -> + OrderingKey = emqx_plugin_libs_rule:proc_tmpl(OrderingKeyTmpl, Data), + Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTmpl, Data), + to_record(OrderingKey, Payload). + +to_record(OrderingKey, Payload) when is_binary(OrderingKey) -> + to_record(binary_to_list(OrderingKey), Payload); +to_record(OrderingKey, Payload) -> + hstreamdb:to_record(OrderingKey, raw, Payload). + +do_append(AfterQuery, Producer, Record) -> + do_append(AfterQuery, false, Producer, Record). + %% TODO: this append is async, remove or change it after we have better disk cache. % do_append(AfterQuery, true, Producer, Record) -> % case hstreamdb:append(Producer, Record) of @@ -221,10 +255,10 @@ do_append(AfterQuery, false, Producer, Record) -> end. client_name(InstId) -> - "backend_hstream_client:" ++ to_string(InstId). + "client:" ++ to_string(InstId). produce_name(ActionId) -> - list_to_atom("backend_hstream_producer:" ++ to_string(ActionId)). + list_to_atom("producer:" ++ to_string(ActionId)). to_string(List) when is_list(List) -> List; to_string(Bin) when is_binary(Bin) -> binary_to_list(Bin); From be6de4aad05d4381bd3ced762ed80d6fac38f84e Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 26 Jul 2022 15:54:58 +0800 Subject: [PATCH 05/71] fix: ee bridge & connector remove app mod --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_app.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 6 ++--- apps/emqx_bridge/src/emqx_bridge_schema.erl | 4 +-- .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 -- .../emqx_ee_bridge/src/emqx_ee_bridge_app.erl | 17 ------------ .../emqx_ee_bridge/src/emqx_ee_bridge_sup.erl | 27 ------------------- lib-ee/emqx_ee_connector/rebar.config | 2 +- .../src/emqx_ee_connector.app.src | 2 -- .../src/emqx_ee_connector_app.erl | 17 ------------ .../src/emqx_ee_connector_sup.erl | 27 ------------------- 11 files changed, 8 insertions(+), 100 deletions(-) delete mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl delete mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 0d3facc5c..e8eb91e57 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -183,7 +183,7 @@ conn_bridge_examples(Method) -> EE = ee_conn_bridge_examples(Method), maps:merge(Broker, EE). --ifdef(EMQX_RELEASE_EDITION). +-if(?EMQX_RELEASE_EDITION == ee). ee_conn_bridge_examples(Method) -> emqx_ee_bridge:conn_bridge_examples(Method). -else. diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 3e67290d6..cac6ab1e6 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -42,7 +42,7 @@ stop(_State) -> ok = emqx_bridge:unload_hook(), ok. --ifdef(EMQX_RELEASE_EDITION). +-if(?EMQX_RELEASE_EDITION == ee). start_ee_apps() -> {ok, _} = application:ensure_all_started(emqx_ee_bridge), {ok, _} = application:ensure_all_started(emqx_ee_connector), diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index c82a290b3..1df6e2b91 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -42,7 +42,7 @@ reset_metrics/1 ]). --ifdef(EMQX_RELEASE_EDITION). +-if(?EMQX_RELEASE_EDITION == ee). bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; bridge_to_resource_type(mqtt) -> emqx_connector_mqtt; bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http; @@ -291,11 +291,11 @@ parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direc parse_confs(Type, Name, Conf) -> parse_enterprise_confs(Type, Name, Conf). --ifdef(EMQX_RELEASE_EDITION). +-if(?EMQX_RELEASE_EDITION == ee). parse_enterprise_confs(Type, Name, Conf) -> emqx_ee_bridge:parse_conf(Type, Name, Conf). -else. -parse_enterprise_confs(Type, Name, Conf) -> +parse_enterprise_confs(Type, Name, _Conf) -> error({not_supported, Type, Name}). -endif. diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index de27cd34c..0b885db8c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -48,7 +48,7 @@ http_schema(Method) -> ExtSchemas = [ref(Module, Method) || Module <- schema_modules()], hoconsc:union(Schemas ++ ExtSchemas). --ifdef(EMQX_RELEASE_EDITION). +-if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> [emqx_bridge_webhook_schema] ++ emqx_ee_bridge:schema_modules(). -else. @@ -158,7 +158,7 @@ fields("node_status") -> {"status", mk(status(), #{})} ]. --ifdef(EMQX_RELEASE_EDITION). +-if(?EMQX_RELEASE_EDITION == ee). ee_fields_bridges() -> emqx_ee_bridge:fields(bridges). -else. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 58846577e..a578b7d0d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,8 +1,6 @@ {application, emqx_ee_bridge, [ - {description, "An OTP application"}, {vsn, "0.1.0"}, {registered, []}, - {mod, {emqx_ee_bridge_app, []}}, {applications, [ kernel, stdlib diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl deleted file mode 100644 index 22268e285..000000000 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_app.erl +++ /dev/null @@ -1,17 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ee_bridge_app). - --behaviour(application). - --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - emqx_ee_bridge_sup:start_link(). - -stop(_State) -> - ok. - -%% internal functions diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl deleted file mode 100644 index 5a2484442..000000000 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sup.erl +++ /dev/null @@ -1,27 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ee_bridge_sup). - --behaviour(supervisor). - --export([start_link/0]). - --export([init/1]). - --define(SERVER, ?MODULE). - -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). - -init([]) -> - SupFlags = #{ - strategy => one_for_all, - intensity => 0, - period => 1 - }, - ChildSpecs = [], - {ok, {SupFlags, ChildSpecs}}. - -%% internal functions diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 1f16e648e..38194cbf5 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,6 +1,6 @@ {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.4"}}} + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 214488f80..6d9fea3c9 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,8 +1,6 @@ {application, emqx_ee_connector, [ - {description, "An OTP application"}, {vsn, "0.1.0"}, {registered, []}, - {mod, {emqx_ee_connector_app, []}}, {applications, [ kernel, stdlib, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl deleted file mode 100644 index 13db2ccb0..000000000 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_app.erl +++ /dev/null @@ -1,17 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ee_connector_app). - --behaviour(application). - --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - emqx_ee_connector_sup:start_link(). - -stop(_State) -> - ok. - -%% internal functions diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl deleted file mode 100644 index f8a219fdc..000000000 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sup.erl +++ /dev/null @@ -1,27 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ee_connector_sup). - --behaviour(supervisor). - --export([start_link/0]). - --export([init/1]). - --define(SERVER, ?MODULE). - -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). - -init([]) -> - SupFlags = #{ - strategy => one_for_all, - intensity => 0, - period => 1 - }, - ChildSpecs = [], - {ok, {SupFlags, ChildSpecs}}. - -%% internal functions From 9ae7c6265609f638936b52b898a96da7e4d26177 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 27 Jul 2022 09:48:39 +0800 Subject: [PATCH 06/71] fix: exs deps & bad suites --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 20 ++++++------------- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 6 +----- .../test/ee_bridge_hstream_SUITE.erl | 16 +++++++++++++++ .../i18n/emqx_ee_connector_hstream.conf | 16 +++++++-------- .../test/ee_connector_hstream_SUITE.erl | 16 +++++++++++++++ mix.exs | 7 +++++-- 7 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl create mode 100644 lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 70550efe4..fe19ed066 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "An OTP application"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 1df6e2b91..cbd337970 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -239,7 +239,7 @@ is_tmp_path(TmpPath, File) -> string:str(str(File), str(TmpPath)) > 0. parse_confs( - webhook, + Type, _Name, #{ url := Url, @@ -248,7 +248,7 @@ parse_confs( headers := Headers, request_timeout := ReqTimeout } = Conf -) -> +) when Type == webhook orelse Type == <<"webhook">> -> {BaseUrl, Path} = parse_url(Url), {ok, BaseUrl2} = emqx_http_lib:uri_parse(BaseUrl), Conf#{ @@ -262,7 +262,7 @@ parse_confs( request_timeout => ReqTimeout } }; -parse_confs(Type = mqtt, Name, #{connector := ConnId, direction := Direction} = Conf) when +parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) when is_binary(ConnId) -> case emqx_connector:parse_connector_id(ConnId) of @@ -278,7 +278,7 @@ parse_confs(Type = mqtt, Name, #{connector := ConnId, direction := Direction} = {_ConnType, _ConnName} -> error({cannot_use_connector_with_different_type, ConnId}) end; -parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when +parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when is_map(ConnectorConfs) -> make_resource_confs( @@ -288,16 +288,8 @@ parse_confs(Type = mqtt, Name, #{connector := ConnectorConfs, direction := Direc Type, Name ); -parse_confs(Type, Name, Conf) -> - parse_enterprise_confs(Type, Name, Conf). - --if(?EMQX_RELEASE_EDITION == ee). -parse_enterprise_confs(Type, Name, Conf) -> - emqx_ee_bridge:parse_conf(Type, Name, Conf). --else. -parse_enterprise_confs(Type, Name, _Conf) -> - error({not_supported, Type, Name}). --endif. +parse_confs(_Type, _Name, Conf) -> + Conf. make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) -> BName = bridge_id(Type, Name), diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 6b2987307..7f051a158 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -9,7 +9,6 @@ schema_modules/0, conn_bridge_examples/1, resource_type/1, - parse_conf/3, fields/1 ]). @@ -27,14 +26,11 @@ conn_bridge_examples(Method) -> resource_type(hstream) -> emqx_ee_connector_hstream; resource_type(<<"hstream">>) -> emqx_ee_connector_hstream. -parse_conf(_Type, _Name, Conf) -> - Conf. - fields(bridges) -> [ {hstream, mk( hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), - #{desc => <<"emqx enterprise config">>} + #{desc => <<"EMQX Enterprise Config">>} )} ]. diff --git a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl new file mode 100644 index 000000000..d03c13cac --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(ee_bridge_hstream_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +%% TODO: diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf index 605944997..fce00baed 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf @@ -2,41 +2,41 @@ emqx_ee_connector_hstream { url { desc { - en: """Hstream Server URL""" + en: """HStream Server URL""" zh: """HStream 服务器 URL""" } label { - en: """Hstream Server URL""" + en: """HStream Server URL""" zh: """HStream 服务器 URL""" } } stream_name { desc { - en: """Hstream Stream Name""" + en: """HStream Stream Name""" zh: """HStream 流名称""" } label { - en: """Hstream Stream Name""" + en: """HStream Stream Name""" zh: """HStream 流名称""" } } ordering_key { desc { - en: """Hstream Ordering Key""" + en: """HStream Ordering Key""" zh: """HStream 分区键""" } label { - en: """Hstream Ordering Key""" + en: """HStream Ordering Key""" zh: """HStream 分区键""" } } pool_size { desc { - en: """Hstream Pool Size""" + en: """HStream Pool Size""" zh: """HStream 连接池大小""" } label { - en: """Hstream Pool Size""" + en: """HStream Pool Size""" zh: """HStream 连接池大小""" } } diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl new file mode 100644 index 000000000..cebe77c6a --- /dev/null +++ b/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(ee_connector_hstream_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +%% TODO: diff --git a/mix.exs b/mix.exs index 714279cfd..2408b68f3 100644 --- a/mix.exs +++ b/mix.exs @@ -88,7 +88,8 @@ defmodule EMQXUmbrella.MixProject do {:ranch, github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd - {:gpb, "4.11.2", override: true, runtime: false} + {:gpb, "4.11.2", override: true, runtime: false}, + {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"} ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() end @@ -215,7 +216,9 @@ defmodule EMQXUmbrella.MixProject do if(edition_type == :enterprise, do: [ emqx_license: :permanent, - emqx_enterprise_conf: :load + emqx_enterprise_conf: :load, + emqx_ee_connector: :permanent, + emqx_ee_bridge: :permanent ], else: [] ) From a4992ef1b52b722b424bae8fe61aa8e22baf0686 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:47:30 +0800 Subject: [PATCH 07/71] fix: hstreamdb connector conf & api --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 4 +- .../emqx_connector/src/emqx_connector_api.erl | 16 +++- .../src/emqx_connector_schema.erl | 24 +++++- .../i18n/emqx_ee_bridge_hstream.conf | 10 +++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 6 +- .../src/emqx_ee_bridge_hstream.erl | 40 +++++----- .../i18n/emqx_ee_connector_hstream.conf | 21 +++++ .../src/emqx_ee_connector.erl | 32 ++++++++ .../src/emqx_ee_connector_hstream.erl | 79 ++++++++++++++----- 9 files changed, 179 insertions(+), 53 deletions(-) create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index cbd337970..6bb30e6b6 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -287,9 +287,7 @@ parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = maps:without([connector, direction], Conf), Type, Name - ); -parse_confs(_Type, _Name, Conf) -> - Conf. + ). make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) -> BName = bridge_id(Type, Name), diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 18409fafd..8dcd3a4aa 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -80,7 +80,7 @@ connector_info_array_example(Method) -> [Config || #{value := Config} <- maps:values(connector_info_examples(Method))]. connector_info_examples(Method) -> - lists:foldl( + Fun = fun(Type, Acc) -> SType = atom_to_list(Type), maps:merge(Acc, #{ @@ -90,9 +90,17 @@ connector_info_examples(Method) -> } }) end, - #{}, - ?CONN_TYPES - ). + Broker = lists:foldl(Fun, #{}, ?CONN_TYPES), + EE = ee_example(Method), + maps:merge(Broker, EE). + +-if(?EMQX_RELEASE_EDITION == ee). +ee_example(Method) -> + emqx_ee_connector:connector_examples(Method). +-else. +ee_example(_Method) -> + #{}. +-endif. info_example(Type, Method) -> maps:merge( diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl index b0f20924f..65d51443b 100644 --- a/apps/emqx_connector/src/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/emqx_connector_schema.erl @@ -44,7 +44,9 @@ post_request() -> http_schema("post"). http_schema(Method) -> - Schemas = [?R_REF(schema_mod(Type), Method) || Type <- ?CONN_TYPES], + Broker = [?R_REF(schema_mod(Type), Method) || Type <- ?CONN_TYPES], + EE = [?R_REF(Module, Method) || Module <- schema_modules()], + Schemas = Broker ++ EE, ?UNION(Schemas). %%====================================================================================== @@ -57,13 +59,29 @@ roots() -> ["connectors"]. fields(connectors) -> fields("connectors"); fields("connectors") -> - [ + Broker = [ {mqtt, ?HOCON( ?MAP(name, ?R_REF(emqx_connector_mqtt_schema, "connector")), #{desc => ?DESC("mqtt")} )} - ]. + ], + EE = ee_fields_connectors(), + Broker ++ EE. + +-if(?EMQX_RELEASE_EDITION == ee). +schema_modules() -> + emqx_ee_connector:schema_modules(). + +ee_fields_connectors() -> + emqx_ee_connector:fields(connectors). +-else. +ee_fields_connectors() -> + []. + +schema_modules() -> + []. +-endif. desc(Record) when Record =:= connectors; diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf index a110dbf27..20b189293 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf @@ -81,4 +81,14 @@ will be forwarded. zh: "Bridge 名字" } } + desc_connector { + desc { + en: """Generic configuration for the connector.""" + zh: """连接器的通用配置。""" + } + label: { + en: "Connector Generic Configuration" + zh: "连接器通用配置。" + } + } } diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 7f051a158..0ed8ee1fe 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -23,12 +23,12 @@ conn_bridge_examples(Method) -> end, lists:foldl(Fun, #{}, schema_modules()). -resource_type(hstream) -> emqx_ee_connector_hstream; -resource_type(<<"hstream">>) -> emqx_ee_connector_hstream. +resource_type(hstreamdb) -> emqx_ee_connector_hstream; +resource_type(<<"hstreamdb">>) -> emqx_ee_connector_hstream. fields(bridges) -> [ - {hstream, + {hstreamdb, mk( hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), #{desc => <<"EMQX Enterprise Config">>} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl index 56bac456e..73f7a20eb 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl @@ -7,7 +7,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include("emqx_ee_bridge.hrl"). --import(hoconsc, [mk/2, enum/1]). +-import(hoconsc, [mk/2, enum/1, ref/2]). -export([ conn_bridge_example/1 @@ -25,7 +25,7 @@ conn_bridge_example(Method) -> #{ - <<"hstream">> => #{ + <<"hstreamdb">> => #{ summary => <<"HStreamDB Bridge">>, value => values(Method) } @@ -35,12 +35,9 @@ values(get) -> maps:merge(values(post), ?METRICS_EXAMPLE); values(post) -> #{ - type => hstream, + type => hstreamdb, name => <<"demo">>, - url => <<"http://127.0.0.1:6570">>, - stream => <<"stream1">>, - ordering_key => <<"some_key">>, - pool_size => 8, + connector => <<"hstreamdb:connector">>, enable => true, direction => egress, local_topic => <<"local/topic/#">>, @@ -56,11 +53,13 @@ namespace() -> "bridge". roots() -> []. fields("config") -> - ExtConfig = [ + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, - {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})} - ], - basic_config() ++ ExtConfig; + {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})}, + {connector, field(connector)} + ]; fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> @@ -68,6 +67,16 @@ fields("put") -> fields("get") -> emqx_bridge_schema:metrics_status_fields() ++ fields("post"). +field(connector) -> + mk( + hoconsc:union([binary(), ref(emqx_ee_connector_hstream, config)]), + #{ + required => true, + example => <<"hstreamdb:demo">>, + desc => ?DESC("desc_connector") + } + ). + desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> @@ -75,17 +84,10 @@ desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> desc(_) -> undefined. -basic_config() -> - Basic = [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})} - ], - emqx_ee_connector_hstream:fields(config) ++ Basic. - %% ------------------------------------------------------------------------------------------------- %% internal type_field() -> - {type, mk(enum([hstream]), #{required => true, desc => ?DESC("desc_type")})}. + {type, mk(enum([hstreamdb]), #{required => true, desc => ?DESC("desc_type")})}. name_field() -> {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf index fce00baed..af9eda92a 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf @@ -1,5 +1,26 @@ emqx_ee_connector_hstream { + type { + desc { + en: "The Connector Type." + zh: "连接器类型。" + } + label: { + en: "Connector Type" + zh: "连接器类型" + } + } + + name { + desc { + en: "Connector name, used as a human-readable description of the connector." + zh: "连接器名称，人类可读的连接器描述。" + } + label: { + en: "Connector Name" + zh: "连接器名称" + } + } url { desc { en: """HStream Server URL""" diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl new file mode 100644 index 000000000..7d175f0eb --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl @@ -0,0 +1,32 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_connector). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + schema_modules/0, + fields/1, + connector_examples/1 +]). + +schema_modules() -> + [emqx_ee_connector_hstream]. + +fields(connectors) -> + [ + {hstreamdb, + mk( + hoconsc:map(name, ref(emqx_ee_connector_hstream, config)), + #{desc => <<"EMQX Enterprise Config">>} + )} + ]. + +connector_examples(Method) -> + Fun = + fun(Module, Examples) -> + Example = erlang:apply(Module, connector_example, [Method]), + maps:merge(Examples, Example) + end, + lists:foldl(Fun, #{}, schema_modules()). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl index fa8439851..00c7e6f61 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl @@ -25,7 +25,8 @@ -export([ roots/0, - fields/1 + fields/1, + connector_example/1 ]). %% ------------------------------------------------------------------------------------------------- @@ -38,7 +39,7 @@ on_stop(InstId, #{client := Client, producer := Producer}) -> StopClientRes = hstreamdb:stop_client(Client), StopProducerRes = hstreamdb:stop_producer(Producer), ?SLOG(info, #{ - msg => "stop hstream connector", + msg => "stop hstreamdb connector", connector => InstId, client => Client, producer => Producer, @@ -64,7 +65,7 @@ on_get_status(_InstId, #{client := Client}) -> end. %% ------------------------------------------------------------------------------------------------- -%% hstream batch callback +%% hstreamdb batch callback %% TODO: maybe remove it after disk cache is ready on_flush_result({{flush, _Stream, _Records}, {ok, _Resp}}) -> @@ -82,9 +83,41 @@ fields(config) -> [ {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, {stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})}, - {ordering_key, mk(binary(), #{required => true, desc => ?DESC("ordering_key")})}, + {ordering_key, mk(binary(), #{required => false, desc => ?DESC("ordering_key")})}, {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})} - ]. + ]; +fields("get") -> + fields("post"); +fields("put") -> + fields(config); +fields("post") -> + [ + {type, mk(hstreamdb, #{required => true, desc => ?DESC("type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("name")})} + ] ++ fields("put"). + +connector_example(Method) -> + #{ + <<"hstreamdb">> => #{ + summary => <<"HStreamDB Connector">>, + value => values(Method) + } + }. + +values(post) -> + maps:merge(values(put), #{name => <<"connector">>}); +values(get) -> + values(post); +values(put) -> + #{ + type => hstreamdb, + url => <<"http://127.0.0.1:6570">>, + stream => <<"stream1">>, + ordering_key => <<"some_key">>, + pool_size => 8 + }; +values(_) -> + #{}. %% ------------------------------------------------------------------------------------------------- %% internal functions @@ -94,7 +127,7 @@ start_client(InstId, Config) -> catch E:R:S -> ?SLOG(error, #{ - msg => "start hstream connector error", + msg => "start hstreamdb connector error", connector => InstId, error => E, reason => R, @@ -104,7 +137,7 @@ start_client(InstId, Config) -> do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> ?SLOG(info, #{ - msg => "starting hstream connector: client", + msg => "starting hstreamdb connector: client", connector => InstId, config => Config }), @@ -118,21 +151,21 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> case is_alive(Client) of true -> ?SLOG(info, #{ - msg => "hstream connector: client started", + msg => "hstreamdb connector: client started", connector => InstId, client => Client }), start_producer(InstId, Client, Config); _ -> ?SLOG(error, #{ - msg => "hstream connector: client not alive", + msg => "hstreamdb connector: client not alive", connector => InstId }), {error, connect_failed} end; {error, {already_started, Pid}} -> ?SLOG(info, #{ - msg => "starting hstream connector: client, find old client. restart client", + msg => "starting hstreamdb connector: client, find old client. restart client", old_client_pid => Pid, old_client_name => ClientName }), @@ -140,7 +173,7 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> start_client(InstId, Config); {error, Error} -> ?SLOG(error, #{ - msg => "hstream connector: client failed", + msg => "hstreamdb connector: client failed", connector => InstId, reason => Error }), @@ -155,7 +188,11 @@ is_alive(Client) -> false end. -start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSize}) -> +start_producer( + InstId, + Client, + Options = #{stream := Stream, pool_size := PoolSize, egress := #{payload := PayloadBin}} +) -> %% TODO: change these batch options after we have better disk cache. BatchSize = maps:get(batch_size, Options, 100), Interval = maps:get(batch_interval, Options, 1000), @@ -168,16 +205,15 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi ], Name = produce_name(InstId), ?SLOG(info, #{ - msg => "starting hstream connector: producer", + msg => "starting hstreamdb connector: producer", connector => InstId }), case hstreamdb:start_producer(Client, Name, ProducerOptions) of {ok, Producer} -> ?SLOG(info, #{ - msg => "hstream connector: producer started" + msg => "hstreamdb connector: producer started" }), EnableBatch = maps:get(enable_batch, Options, false), - PayloadBin = maps:get(payload, Options, <<"">>), Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin), OrderingKeyBin = maps:get(ordering_key, Options, <<"">>), OrderingKey = emqx_plugin_libs_rule:preproc_tmpl(OrderingKeyBin), @@ -191,7 +227,8 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi {ok, State}; {error, {already_started, Pid}} -> ?SLOG(info, #{ - msg => "starting hstream connector: producer, find old producer. restart producer", + msg => + "starting hstreamdb connector: producer, find old producer. restart producer", old_producer_pid => Pid, old_producer_name => Name }), @@ -199,7 +236,7 @@ start_producer(InstId, Client, Options = #{stream := Stream, pool_size := PoolSi start_producer(InstId, Client, Options); {error, Reason} -> ?SLOG(error, #{ - msg => "starting hstream connector: producer, failed", + msg => "starting hstreamdb connector: producer, failed", reason => Reason }), {error, Reason} @@ -223,13 +260,13 @@ do_append(AfterQuery, Producer, Record) -> % case hstreamdb:append(Producer, Record) of % ok -> % ?SLOG(debug, #{ -% msg => "hstream producer async append success", +% msg => "hstreamdb producer async append success", % record => Record % }), % emqx_resource:query_success(AfterQuery); % {error, Reason} -> % ?SLOG(error, #{ -% msg => "hstream producer async append failed", +% msg => "hstreamdb producer async append failed", % reason => Reason, % record => Record % }), @@ -241,13 +278,13 @@ do_append(AfterQuery, false, Producer, Record) -> case hstreamdb:append_flush(Producer, Record) of {ok, _} -> ?SLOG(debug, #{ - msg => "hstream producer sync append success", + msg => "hstreamdb producer sync append success", record => Record }), emqx_resource:query_success(AfterQuery); {error, Reason} -> ?SLOG(error, #{ - msg => "hstream producer sync append failed", + msg => "hstreamdb producer sync append failed", reason => Reason, record => Record }), From b7c245c5b0074b607456dc839503c69d8d2ca2cf Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:50:34 +0800 Subject: [PATCH 08/71] fix: hstreamdb zh docs --- .../i18n/emqx_ee_bridge_hstream.conf | 18 +++++------ .../i18n/emqx_ee_connector_hstream.conf | 32 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf index 20b189293..ad07ad377 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf @@ -31,32 +31,32 @@ will be forwarded. config_enable { desc { en: """Enable or disable this bridge""" - zh: """启用/禁用 Bridge""" + zh: """启用/禁用桥接""" } label { en: "Enable Or Disable Bridge" - zh: "启用/禁用 Bridge" + zh: "启用/禁用桥接" } } config_direction { desc { en: """The direction of this bridge, MUST be 'egress'""" - zh: """Bridge 的方向， 必须是 egress""" + zh: """桥接的方向， 必须是 egress""" } label { en: "Bridge Direction" - zh: "Bridge 方向" + zh: "桥接方向" } } desc_config { desc { en: """Configuration for an HStreamDB bridge.""" - zh: """HStreamDB Bridge 配置""" + zh: """HStreamDB 桥接配置""" } label: { en: "HStreamDB Bridge Configuration" - zh: "HStreamDB Bridge 配置" + zh: "HStreamDB 桥接配置" } } @@ -67,18 +67,18 @@ will be forwarded. } label { en: "Bridge Type" - zh: "Bridge 类型" + zh: "桥接类型" } } desc_name { desc { en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """Bridge 名字，Bridge 的可读描述""" + zh: """桥接名字，可读描述""" } label { en: "Bridge Name" - zh: "Bridge 名字" + zh: "桥接名字" } } desc_connector { diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf index af9eda92a..080076065 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf @@ -23,42 +23,42 @@ emqx_ee_connector_hstream { } url { desc { - en: """HStream Server URL""" - zh: """HStream 服务器 URL""" + en: """HStreamDB Server URL""" + zh: """HStreamDB 服务器 URL""" } label { - en: """HStream Server URL""" - zh: """HStream 服务器 URL""" + en: """HStreamDB Server URL""" + zh: """HStreamDB 服务器 URL""" } } stream_name { desc { - en: """HStream Stream Name""" - zh: """HStream 流名称""" + en: """HStreamDB Stream Name""" + zh: """HStreamDB 流名称""" } label { - en: """HStream Stream Name""" - zh: """HStream 流名称""" + en: """HStreamDB Stream Name""" + zh: """HStreamDB 流名称""" } } ordering_key { desc { - en: """HStream Ordering Key""" - zh: """HStream 分区键""" + en: """HStreamDB Ordering Key""" + zh: """HStreamDB 分区键""" } label { - en: """HStream Ordering Key""" - zh: """HStream 分区键""" + en: """HStreamDB Ordering Key""" + zh: """HStreamDB 分区键""" } } pool_size { desc { - en: """HStream Pool Size""" - zh: """HStream 连接池大小""" + en: """HStreamDB Pool Size""" + zh: """HStreamDB 连接池大小""" } label { - en: """HStream Pool Size""" - zh: """HStream 连接池大小""" + en: """HStreamDB Pool Size""" + zh: """HStreamDB 连接池大小""" } } } From 01e9e8a0eb19ad25992cf6126904e7c6968047c5 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 29 Jul 2022 14:10:01 +0800 Subject: [PATCH 09/71] feat: add ee influxdb connector & bridge --- .../src/emqx_connector_schema.erl | 8 +- .../i18n/emqx_ee_bridge_hstream.conf | 8 +- .../i18n/emqx_ee_bridge_influxdb.conf | 94 +++++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 10 +- .../src/emqx_ee_bridge_influxdb.erl | 99 ++++++++ .../i18n/emqx_ee_connector_influxdb.conf | 165 ++++++++++++ .../src/emqx_ee_connector.erl | 34 ++- .../src/emqx_ee_connector_hstream.erl | 16 +- .../src/emqx_ee_connector_influxdb.erl | 240 ++++++++++++++++++ 9 files changed, 650 insertions(+), 24 deletions(-) create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl index 65d51443b..f0c9479de 100644 --- a/apps/emqx_connector/src/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/emqx_connector_schema.erl @@ -45,7 +45,7 @@ post_request() -> http_schema(Method) -> Broker = [?R_REF(schema_mod(Type), Method) || Type <- ?CONN_TYPES], - EE = [?R_REF(Module, Method) || Module <- schema_modules()], + EE = ee_schemas(Method), Schemas = Broker ++ EE, ?UNION(Schemas). @@ -70,8 +70,8 @@ fields("connectors") -> Broker ++ EE. -if(?EMQX_RELEASE_EDITION == ee). -schema_modules() -> - emqx_ee_connector:schema_modules(). +ee_schemas(Method) -> + emqx_ee_connector:api_schemas(Method). ee_fields_connectors() -> emqx_ee_connector:fields(connectors). @@ -79,7 +79,7 @@ ee_fields_connectors() -> ee_fields_connectors() -> []. -schema_modules() -> +ee_schemas(_) -> []. -endif. diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf index ad07ad377..2e4397a04 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf @@ -13,10 +13,10 @@ will be forwarded. 注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。 """ } - label { - en: "Local Topic" - zh: "本地 Topic" - } + label { + en: "Local Topic" + zh: "本地 Topic" + } } payload { desc { diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf new file mode 100644 index 000000000..f421fb912 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf @@ -0,0 +1,94 @@ +emqx_ee_bridge_influxdb { + local_topic { + desc { + en: """ +The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded. +""" + zh: """ +发送到 'local_topic' 的消息都会转发到 InfluxDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。 +""" + } + label { + en: "Local Topic" + zh: "本地 Topic" + } + } + payload { + desc { + en: """The payload to be forwarded to the InfluxDB. Placeholders supported.""" + zh: """要转发到 InfluxDB 的数据内容，支持占位符""" + } + label { + en: "Payload" + zh: "消息内容" + } + } + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用桥接""" + } + label { + en: "Enable Or Disable Bridge" + zh: "启用/禁用桥接" + } + } + config_direction { + desc { + en: """The direction of this bridge, MUST be 'egress'""" + zh: """桥接的方向， 必须是 egress""" + } + label { + en: "Bridge Direction" + zh: "桥接方向" + } + } + + desc_config { + desc { + en: """Configuration for an InfluxDB bridge.""" + zh: """InfluxDB 桥接配置""" + } + label: { + en: "InfluxDB Bridge Configuration" + zh: "InfluxDB 桥接配置" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: "Bridge Type" + zh: "桥接类型" + } + } + + desc_name { + desc { + en: """Bridge name, used as a human-readable description of the bridge.""" + zh: """桥接名字，可读描述""" + } + label { + en: "Bridge Name" + zh: "桥接名字" + } + } + desc_connector { + desc { + en: """Generic configuration for the connector.""" + zh: """连接器的通用配置。""" + } + label: { + en: "Connector Generic Configuration" + zh: "连接器通用配置。" + } + } +} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 0ed8ee1fe..ff9661cc8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -13,7 +13,7 @@ ]). schema_modules() -> - [emqx_ee_bridge_hstream]. + [emqx_ee_bridge_hstream, emqx_ee_bridge_influxdb]. conn_bridge_examples(Method) -> Fun = @@ -23,8 +23,9 @@ conn_bridge_examples(Method) -> end, lists:foldl(Fun, #{}, schema_modules()). +resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); resource_type(hstreamdb) -> emqx_ee_connector_hstream; -resource_type(<<"hstreamdb">>) -> emqx_ee_connector_hstream. +resource_type(influxdb) -> emqx_ee_connector_influxdb. fields(bridges) -> [ @@ -32,5 +33,10 @@ fields(bridges) -> mk( hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), #{desc => <<"EMQX Enterprise Config">>} + )}, + {influxdb, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_influxdb, "config")), + #{desc => <<"EMQX Enterprise Config">>} )} ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl new file mode 100644 index 000000000..d285c2621 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_influxdb). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_ee_bridge.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_example/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_example(Method) -> + #{ + <<"influxdb">> => #{ + summary => <<"InfluxDB Bridge">>, + value => values(Method) + } + }. + +values(get) -> + maps:merge(values(post), ?METRICS_EXAMPLE); +values(post) -> + #{ + type => influxdb, + name => <<"demo">>, + connector => <<"influxdb:api_v2_connector">>, + enable => true, + direction => egress, + local_topic => <<"local/topic/#">>, + payload => <<"${payload}">> + }; +values(put) -> + values(post). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, + {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, + {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})}, + {connector, field(connector)} + ]; +fields("post") -> + [type_field(), name_field() | fields("config")]; +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:metrics_status_fields() ++ fields("post"). + +field(connector) -> + ConnectorConfigRef = + [ + ref(emqx_ee_connector_influxdb, udp), + ref(emqx_ee_connector_influxdb, api_v1), + ref(emqx_ee_connector_influxdb, api_v2) + ], + mk( + hoconsc:union([binary() | ConnectorConfigRef]), + #{ + required => true, + example => <<"influxdb:demo">>, + desc => ?DESC("desc_connector") + } + ). + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for HStream using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- +%% internal +type_field() -> + {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf new file mode 100644 index 000000000..a909c9a72 --- /dev/null +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf @@ -0,0 +1,165 @@ + +emqx_ee_connector_influxdb { + type { + desc { + en: """The Connector Type.""" + zh: """连接器类型。""" + } + label: { + en: """Connector Type""" + zh: """连接器类型""" + } + } + + name { + desc { + en: """Connector name, used as a human-readable description of the connector.""" + zh: """连接器名称，人类可读的连接器描述。""" + } + label: { + en: """Connector Name""" + zh: """连接器名称""" + } + } + host { + desc { + en: """InfluxDB host.""" + zh: """InfluxDB 主机地址。""" + } + label: { + en: """Host""" + zh: """主机""" + } + } + port { + desc { + en: """InfluxDB port.""" + zh: """InfluxDB 端口。""" + } + label: { + en: """Port""" + zh: """端口""" + } + } + protocol { + desc { + en: """InfluxDB protocol. UDP or HTTP API or HTTP API V2""" + zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2""" + } + label: { + en: """Protocol""" + zh: """协议""" + } + } + protocol_udp { + desc { + en: """InfluxDB protocol.""" + zh: """InfluxDB UDP 协议""" + } + label: { + en: """UDP Protocol""" + zh: """UDP 协议""" + } + } + protocol_api_v1 { + desc { + en: """InfluxDB protocol. Support InfluxDB v1.8 and before.""" + zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本""" + } + label: { + en: """HTTP API Protocol""" + zh: """HTTP API 协议""" + } + } + protocol_api_v2 { + desc { + en: """InfluxDB protocol. Support InfluxDB v2.0 and after.""" + zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本""" + } + label: { + en: """HTTP API V2 Protocol""" + zh: """HTTP API V2 协议""" + } + } + database { + desc { + en: """InfluxDB database.""" + zh: """InfluxDB 数据库。""" + } + label: { + en: "Database" + zh: "数据库" + } + } + username { + desc { + en: "InfluxDB username." + zh: "InfluxDB 用户名。" + } + label: { + en: "Username" + zh: "用户名" + } + } + password { + desc { + en: "InfluxDB password." + zh: "InfluxDB 密码。" + } + label: { + en: "Password" + zh: "密码" + } + } + bucket { + desc { + en: "InfluxDB bucket name." + zh: "InfluxDB bucket 名称" + } + label: { + en: "Bucket" + zh: "Bucket" + } + } + org { + desc { + en: """InfluxDB organization name.""" + zh: """InfluxDB 组织名称。""" + } + label: { + en: """Organization""" + zh: """组织""" + } + } + token { + desc { + en: """InfluxDB token.""" + zh: """InfluxDB token。""" + } + label: { + en: """Token""" + zh: """Token""" + } + } + precision { + desc { + en: """InfluxDB time precision.""" + zh: """InfluxDB 时间精度。""" + } + label: { + en: """Time Precision""" + zh: """时间精度""" + } + } + pool_size { + desc { + en: """InfluxDB Pool Size""" + zh: """InfluxDB 连接池大小""" + } + label { + en: """InfluxDB Pool Size""" + zh: """InfluxDB 连接池大小""" + } + } + +} diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl index 7d175f0eb..d0884f945 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl @@ -6,13 +6,18 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -export([ - schema_modules/0, fields/1, - connector_examples/1 + connector_examples/1, + api_schemas/1 ]). -schema_modules() -> - [emqx_ee_connector_hstream]. +api_schemas(Method) -> + [ + ref(emqx_ee_connector_hstream, Method), + ref(emqx_ee_connector_influxdb, Method ++ "_udp"), + ref(emqx_ee_connector_influxdb, Method ++ "_api_v1"), + ref(emqx_ee_connector_influxdb, Method ++ "_api_v2") + ]. fields(connectors) -> [ @@ -21,12 +26,27 @@ fields(connectors) -> hoconsc:map(name, ref(emqx_ee_connector_hstream, config)), #{desc => <<"EMQX Enterprise Config">>} )} + ] ++ fields(influxdb); +fields(influxdb) -> + [ + {Protocol, + mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, Protocol)), #{ + desc => <<"EMQX Enterprise Config">> + })} + || Protocol <- [udp, api_v1, api_v2] ]. connector_examples(Method) -> - Fun = - fun(Module, Examples) -> - Example = erlang:apply(Module, connector_example, [Method]), + MergeFun = + fun(Example, Examples) -> maps:merge(Examples, Example) end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, connector_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, lists:foldl(Fun, #{}, schema_modules()). + +schema_modules() -> + [emqx_ee_connector_hstream, emqx_ee_connector_influxdb]. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl index 00c7e6f61..7a8ce3fe9 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl @@ -26,7 +26,7 @@ -export([ roots/0, fields/1, - connector_example/1 + connector_examples/1 ]). %% ------------------------------------------------------------------------------------------------- @@ -96,13 +96,15 @@ fields("post") -> {name, mk(binary(), #{required => true, desc => ?DESC("name")})} ] ++ fields("put"). -connector_example(Method) -> - #{ - <<"hstreamdb">> => #{ - summary => <<"HStreamDB Connector">>, - value => values(Method) +connector_examples(Method) -> + [ + #{ + <<"hstreamdb">> => #{ + summary => <<"HStreamDB Connector">>, + value => values(Method) + } } - }. + ]. values(post) -> maps:merge(values(put), #{name => <<"connector">>}); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl new file mode 100644 index 000000000..0ed7964b7 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -0,0 +1,240 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_connector_influxdb). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-behaviour(emqx_resource). + +-define(PUT_FIELDS_FILTER, fun({Name, _}) -> not lists:member(Name, [type, name]) end). + +%% callbacks of behaviour emqx_resource +-export([ + on_start/2, + on_stop/2, + on_query/4, + on_get_status/2 +]). + +-export([ + fields/1, + connector_examples/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% resource callback + +on_start(InstId, Config) -> + start_client(InstId, Config). + +on_stop(_InstId, _State) -> + ok. + +on_query(_InstId, {send_message, _Data}, _AfterQuery, _State) -> + ok. + +on_get_status(_InstId, _State) -> + % connected; + disconnected. + +%% ------------------------------------------------------------------------------------------------- +%% schema + +fields("put_udp") -> + lists:filter(?PUT_FIELDS_FILTER, fields(udp)); +fields("put_api_v1") -> + lists:filter(?PUT_FIELDS_FILTER, fields(api_v1)); +fields("put_api_v2") -> + lists:filter(?PUT_FIELDS_FILTER, fields(api_v2)); +fields("get_udp") -> + fields(udp); +fields("get_api_v1") -> + fields(api_v1); +fields("get_api_v2") -> + fields(api_v2); +fields("post_udp") -> + fields(udp); +fields("post_api_v1") -> + fields(api_v1); +fields("post_api_v2") -> + fields(api_v2); +fields(basic) -> + [ + {host, + mk(binary(), #{required => true, default => <<"120.0.0.1">>, desc => ?DESC("host")})}, + {port, mk(pos_integer(), #{required => true, default => 8086, desc => ?DESC("port")})}, + {precision, + mk(enum([ns, us, ms, s, m, h]), #{ + required => false, default => ms, desc => ?DESC("precision") + })}, + {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}, + {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("name")})} + ]; +fields(udp) -> + [ + {protocol, mk(enum([udp]), #{required => true, desc => ?DESC("protocol_udp")})} + ] ++ fields(basic); +fields(api_v1) -> + [ + {protocol, mk(enum([api_v1]), #{required => true, desc => ?DESC("protocol_api_v1")})}, + {database, mk(binary(), #{required => true, desc => ?DESC("database")})}, + {username, mk(binary(), #{required => true, desc => ?DESC("username")})}, + {password, mk(binary(), #{required => true, desc => ?DESC("password")})} + ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic); +fields(api_v2) -> + [ + {protocol, mk(enum([api_v2]), #{required => true, desc => ?DESC("protocol_api_v2")})}, + {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})}, + {org, mk(binary(), #{required => true, desc => ?DESC("org")})}, + {token, mk(binary(), #{required => true, desc => ?DESC("token")})} + ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic). + +connector_examples(Method) -> + [ + #{ + <<"influxdb_udp">> => #{ + summary => <<"InfluxDB UDP Connector">>, + value => values(udp, Method) + } + }, + #{ + <<"influxdb_api_v1">> => #{ + summary => <<"InfluxDB HTTP API V1 Connector">>, + value => values(api_v1, Method) + } + }, + #{ + <<"influxdb_api_v2">> => #{ + summary => <<"InfluxDB HTTP API V2 Connector">>, + value => values(api_v2, Method) + } + } + ]. + +values(Protocol, get) -> + values(Protocol, post); +values(Protocol, post) -> + ConnectorName = list_to_binary(io_lib:format("~p_connector", [Protocol])), + maps:merge(values(Protocol, put), #{type => influxdb, name => ConnectorName}); +values(udp, put) -> + #{ + protocol => udp, + host => <<"127.0.0.1">>, + port => 8089, + precision => ms, + pool_size => 8 + }; +values(api_v1, put) -> + #{ + protocol => api_v1, + host => <<"127.0.0.1">>, + port => 8086, + precision => ms, + pool_size => 8, + database => <<"my_db">>, + username => <<"my_user">>, + password => <<"my_password">>, + ssl => #{enable => false} + }; +values(api_v2, put) -> + #{ + protocol => api_v2, + host => <<"127.0.0.1">>, + port => 8086, + precision => ms, + pool_size => 8, + bucket => <<"my_bucket">>, + org => <<"my_org">>, + token => <<"my_token">>, + ssl => #{enable => false} + }. +%% ------------------------------------------------------------------------------------------------- +%% internal functions + +start_client(InstId, Config) -> + io:format("InstId ~p~n", [InstId]), + client_config(InstId, Config). + +% ClientConfig = client_config(InstId, Config), +% case influxdb:start_client(ClientConfig) of +% {ok, Client} -> +% true = influxdb:is_alive(Client), +% maybe_pool_size(Client, Params); +% {error, {already_started, Client0}} -> +% _ = influxdb:stop_client(Client0), +% {ok, Client} = influxdb:start_client(Options), +% true = influxdb:is_alive(Client), +% maybe_pool_size(Client, Params); +% {error, Reason} -> +% logger:log(error, "Initiate influxdb failed ~0p", [Reason]), +% error({start_pool_failed, ResId}) +% end. + +client_config( + _InstId, + Config = #{ + host := Host, + port := Port, + pool_size := PoolSize + } +) -> + [ + {host, Host}, + {port, Port}, + {pool_size, PoolSize}, + {pool, atom_pname_todo}, + {precision, maps:get(precision, Config, ms)} + ] ++ protocol_config(Config). + +protocol_config(#{ + protocol := udp +}) -> + [ + {protocol, udp} + ]; +protocol_config(#{ + protocol := api_v1, + username := Username, + password := Password, + database := DB, + ssl := SSL +}) -> + [ + {protocol, http}, + {version, v1}, + {username, Username}, + {password, Password}, + {database, DB}, + {ssl, SSL} + ] ++ ssl_config(SSL); +protocol_config(#{ + protocol := api_v2, + bucket := Bucket, + org := Org, + token := Token, + ssl := SSL +}) -> + [ + {protocol, http}, + {version, v2}, + {bucket, Bucket}, + {org, Org}, + {token, Token}, + {ssl, SSL} + ] ++ ssl_config(SSL). + +ssl_config(#{enable := false}) -> + [ + {https_enabled, false} + ]; +ssl_config(SSL = #{enable := true}) -> + [ + {https_enabled, true}, + {transport, ssl} + ] ++ maps:to_list(maps:remove(enable, SSL)). From 4c7ca2217ce53d5bb94e71b0b4b72e139d27687a Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 29 Jul 2022 18:36:48 +0800 Subject: [PATCH 10/71] fix: add influxdb udp api_v1 api_v2 connector --- .../src/emqx_ee_bridge_influxdb.erl | 6 +- lib-ee/emqx_ee_connector/rebar.config | 3 +- .../src/emqx_ee_connector.erl | 13 +- .../src/emqx_ee_connector_influxdb.erl | 124 ++++++++++++------ mix.exs | 6 + 5 files changed, 102 insertions(+), 50 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index d285c2621..7004b70a1 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -70,9 +70,9 @@ fields("get") -> field(connector) -> ConnectorConfigRef = [ - ref(emqx_ee_connector_influxdb, udp), - ref(emqx_ee_connector_influxdb, api_v1), - ref(emqx_ee_connector_influxdb, api_v2) + ref(emqx_ee_connector_influxdb, influxdb_udp), + ref(emqx_ee_connector_influxdb, influxdb_api_v1), + ref(emqx_ee_connector_influxdb, influxdb_api_v2) ], mk( hoconsc:union([binary() | ConnectorConfigRef]), diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 38194cbf5..0a3f6866b 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,6 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}} + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.2"}}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl index d0884f945..55ccc2c2c 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl @@ -26,14 +26,17 @@ fields(connectors) -> hoconsc:map(name, ref(emqx_ee_connector_hstream, config)), #{desc => <<"EMQX Enterprise Config">>} )} - ] ++ fields(influxdb); + ]; +% ] ++ fields(influxdb); fields(influxdb) -> [ - {Protocol, - mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, Protocol)), #{ + { + influxdb, + mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, influxdb_udp)), #{ desc => <<"EMQX Enterprise Config">> - })} - || Protocol <- [udp, api_v1, api_v2] + }) + } + % || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2] ]. connector_examples(Method) -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 0ed7964b7..1efc0b263 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -32,37 +32,41 @@ on_start(InstId, Config) -> start_client(InstId, Config). -on_stop(_InstId, _State) -> - ok. +on_stop(_InstId, #{client := Client}) -> + influxdb:stop_client(Client). on_query(_InstId, {send_message, _Data}, _AfterQuery, _State) -> ok. -on_get_status(_InstId, _State) -> - % connected; - disconnected. +on_get_status(_InstId, #{client := Client}) -> + case influxdb:is_alive(Client) of + true -> + connected; + false -> + disconnected + end. %% ------------------------------------------------------------------------------------------------- %% schema fields("put_udp") -> - lists:filter(?PUT_FIELDS_FILTER, fields(udp)); + lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_udp)); fields("put_api_v1") -> - lists:filter(?PUT_FIELDS_FILTER, fields(api_v1)); + lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v1)); fields("put_api_v2") -> - lists:filter(?PUT_FIELDS_FILTER, fields(api_v2)); + lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v2)); fields("get_udp") -> - fields(udp); + fields(influxdb_udp); fields("get_api_v1") -> - fields(api_v1); + fields(influxdb_api_v1); fields("get_api_v2") -> - fields(api_v2); + fields(influxdb_api_v2); fields("post_udp") -> - fields(udp); + fields(influxdb_udp); fields("post_api_v1") -> - fields(api_v1); + fields(influxdb_api_v1); fields("post_api_v2") -> - fields(api_v2); + fields(influxdb_api_v2); fields(basic) -> [ {host, @@ -73,23 +77,22 @@ fields(basic) -> required => false, default => ms, desc => ?DESC("precision") })}, {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}, - {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("type")})}, {name, mk(binary(), #{required => true, desc => ?DESC("name")})} ]; -fields(udp) -> +fields(influxdb_udp) -> [ - {protocol, mk(enum([udp]), #{required => true, desc => ?DESC("protocol_udp")})} + {type, mk(influxdb_udp, #{required => true, desc => ?DESC("type")})} ] ++ fields(basic); -fields(api_v1) -> +fields(influxdb_api_v1) -> [ - {protocol, mk(enum([api_v1]), #{required => true, desc => ?DESC("protocol_api_v1")})}, + {type, mk(influxdb_api_v1, #{required => true, desc => ?DESC("type")})}, {database, mk(binary(), #{required => true, desc => ?DESC("database")})}, {username, mk(binary(), #{required => true, desc => ?DESC("username")})}, {password, mk(binary(), #{required => true, desc => ?DESC("password")})} ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic); -fields(api_v2) -> +fields(influxdb_api_v2) -> [ - {protocol, mk(enum([api_v2]), #{required => true, desc => ?DESC("protocol_api_v2")})}, + {type, mk(influxdb_api_v2, #{required => true, desc => ?DESC("type")})}, {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})}, {org, mk(binary(), #{required => true, desc => ?DESC("org")})}, {token, mk(binary(), #{required => true, desc => ?DESC("token")})} @@ -120,11 +123,11 @@ connector_examples(Method) -> values(Protocol, get) -> values(Protocol, post); values(Protocol, post) -> + Type = list_to_atom(io_lib:format("influxdb_~p", [Protocol])), ConnectorName = list_to_binary(io_lib:format("~p_connector", [Protocol])), - maps:merge(values(Protocol, put), #{type => influxdb, name => ConnectorName}); + maps:merge(values(Protocol, put), #{type => Type, name => ConnectorName}); values(udp, put) -> #{ - protocol => udp, host => <<"127.0.0.1">>, port => 8089, precision => ms, @@ -132,7 +135,6 @@ values(udp, put) -> }; values(api_v1, put) -> #{ - protocol => api_v1, host => <<"127.0.0.1">>, port => 8086, precision => ms, @@ -144,7 +146,6 @@ values(api_v1, put) -> }; values(api_v2, put) -> #{ - protocol => api_v2, host => <<"127.0.0.1">>, port => 8086, precision => ms, @@ -158,23 +159,64 @@ values(api_v2, put) -> %% internal functions start_client(InstId, Config) -> - io:format("InstId ~p~n", [InstId]), - client_config(InstId, Config). + ClientConfig = client_config(InstId, Config), + ?SLOG(info, #{ + msg => "starting influxdb connector", + connector => InstId, + config => Config, + client_config => ClientConfig + }), + try + do_start_client(InstId, ClientConfig, Config) + catch + E:R:S -> + ?SLOG(error, #{ + msg => "start influxdb connector error", + connector => InstId, + error => E, + reason => R, + stack => S + }), + {error, R} + end. -% ClientConfig = client_config(InstId, Config), -% case influxdb:start_client(ClientConfig) of -% {ok, Client} -> -% true = influxdb:is_alive(Client), -% maybe_pool_size(Client, Params); -% {error, {already_started, Client0}} -> -% _ = influxdb:stop_client(Client0), -% {ok, Client} = influxdb:start_client(Options), -% true = influxdb:is_alive(Client), -% maybe_pool_size(Client, Params); -% {error, Reason} -> -% logger:log(error, "Initiate influxdb failed ~0p", [Reason]), -% error({start_pool_failed, ResId}) -% end. +do_start_client(InstId, ClientConfig, Config = #{egress := #{payload := PayloadBin}}) -> + case influxdb:start_client(ClientConfig) of + {ok, Client} -> + case influxdb:is_alive(Client) of + true -> + Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin), + ?SLOG(info, #{ + msg => "starting influxdb connector success", + connector => InstId, + client => Client + }), + #{client => Client, payload => Payload}; + false -> + ?SLOG(error, #{ + msg => "starting influxdb connector failed", + connector => InstId, + client => Client, + reason => "client is not alive" + }), + {error, influxdb_client_not_alive} + end; + {error, {already_started, Client0}} -> + ?SLOG(info, #{ + msg => "starting influxdb connector,find already started client", + connector => InstId, + old_client => Client0 + }), + _ = influxdb:stop_client(Client0), + do_start_client(InstId, ClientConfig, Config); + {error, Reason} -> + ?SLOG(error, #{ + msg => "starting influxdb connector failed", + connector => InstId, + reason => Reason + }), + {error, Reason} + end. client_config( _InstId, diff --git a/mix.exs b/mix.exs index 6648aa35e..039172eb9 100644 --- a/mix.exs +++ b/mix.exs @@ -89,9 +89,15 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.11.2", override: true, runtime: false}, +<<<<<<< HEAD {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"} ] ++ umbrella_apps() ++ enterprise_apps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() +======= + {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, + {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.2"} + ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() +>>>>>>> fix: add influxdb udp api_v1 api_v2 connector end defp umbrella_apps() do From fa54bf56128d44f7f64056b387dc69dd7741335d Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Sun, 31 Jul 2022 15:33:57 +0800 Subject: [PATCH 11/71] fix: influxdb connector api available --- .../src/emqx_ee_bridge_influxdb.erl | 14 ++---- .../src/emqx_ee_connector.erl | 14 +++--- .../src/emqx_ee_connector_influxdb.erl | 50 ++++++++++--------- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 7004b70a1..dca55ba23 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -61,7 +61,11 @@ fields("config") -> {connector, field(connector)} ]; fields("post") -> - [type_field(), name_field() | fields("config")]; + [ + {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} + | fields("config") + ]; fields("put") -> fields("config"); fields("get") -> @@ -89,11 +93,3 @@ desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for HStream using `", string:to_upper(Method), "` method."]; desc(_) -> undefined. - -%% ------------------------------------------------------------------------------------------------- -%% internal -type_field() -> - {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})}. - -name_field() -> - {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl index 55ccc2c2c..9e8e7bac1 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl @@ -26,17 +26,16 @@ fields(connectors) -> hoconsc:map(name, ref(emqx_ee_connector_hstream, config)), #{desc => <<"EMQX Enterprise Config">>} )} - ]; -% ] ++ fields(influxdb); + ] ++ fields(influxdb); fields(influxdb) -> [ { - influxdb, - mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, influxdb_udp)), #{ + Protocol, + mk(hoconsc:map(name, ref(emqx_ee_connector_influxdb, Protocol)), #{ desc => <<"EMQX Enterprise Config">> }) } - % || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2] + || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2] ]. connector_examples(Method) -> @@ -52,4 +51,7 @@ connector_examples(Method) -> lists:foldl(Fun, #{}, schema_modules()). schema_modules() -> - [emqx_ee_connector_hstream, emqx_ee_connector_influxdb]. + [ + emqx_ee_connector_hstream, + emqx_ee_connector_influxdb + ]. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 1efc0b263..37915d2e8 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -11,8 +11,6 @@ -behaviour(emqx_resource). --define(PUT_FIELDS_FILTER, fun({Name, _}) -> not lists:member(Name, [type, name]) end). - %% callbacks of behaviour emqx_resource -export([ on_start/2, @@ -50,23 +48,29 @@ on_get_status(_InstId, #{client := Client}) -> %% schema fields("put_udp") -> - lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_udp)); + fields(influxdb_udp); fields("put_api_v1") -> - lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v1)); + fields(influxdb_api_v1); fields("put_api_v2") -> - lists:filter(?PUT_FIELDS_FILTER, fields(influxdb_api_v2)); + fields(influxdb_api_v2); fields("get_udp") -> - fields(influxdb_udp); + Key = influxdb_udp, + fields(Key) ++ type_name_field(Key); fields("get_api_v1") -> - fields(influxdb_api_v1); + Key = influxdb_api_v1, + fields(Key) ++ type_name_field(Key); fields("get_api_v2") -> - fields(influxdb_api_v2); + Key = influxdb_api_v2, + fields(Key) ++ type_name_field(Key); fields("post_udp") -> - fields(influxdb_udp); + Key = influxdb_udp, + fields(Key) ++ type_name_field(Key); fields("post_api_v1") -> - fields(influxdb_api_v1); + Key = influxdb_api_v1, + fields(Key) ++ type_name_field(Key); fields("post_api_v2") -> - fields(influxdb_api_v2); + Key = influxdb_api_v2, + fields(Key) ++ type_name_field(Key); fields(basic) -> [ {host, @@ -76,28 +80,29 @@ fields(basic) -> mk(enum([ns, us, ms, s, m, h]), #{ required => false, default => ms, desc => ?DESC("precision") })}, - {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}, - {name, mk(binary(), #{required => true, desc => ?DESC("name")})} + {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})} ]; fields(influxdb_udp) -> - [ - {type, mk(influxdb_udp, #{required => true, desc => ?DESC("type")})} - ] ++ fields(basic); + fields(basic); fields(influxdb_api_v1) -> [ - {type, mk(influxdb_api_v1, #{required => true, desc => ?DESC("type")})}, {database, mk(binary(), #{required => true, desc => ?DESC("database")})}, {username, mk(binary(), #{required => true, desc => ?DESC("username")})}, {password, mk(binary(), #{required => true, desc => ?DESC("password")})} ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic); fields(influxdb_api_v2) -> [ - {type, mk(influxdb_api_v2, #{required => true, desc => ?DESC("type")})}, {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})}, {org, mk(binary(), #{required => true, desc => ?DESC("org")})}, {token, mk(binary(), #{required => true, desc => ?DESC("token")})} ] ++ emqx_connector_schema_lib:ssl_fields() ++ fields(basic). +type_name_field(Type) -> + [ + {type, mk(Type, #{required => true, desc => ?DESC("type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("name")})} + ]. + connector_examples(Method) -> [ #{ @@ -123,9 +128,8 @@ connector_examples(Method) -> values(Protocol, get) -> values(Protocol, post); values(Protocol, post) -> - Type = list_to_atom(io_lib:format("influxdb_~p", [Protocol])), - ConnectorName = list_to_binary(io_lib:format("~p_connector", [Protocol])), - maps:merge(values(Protocol, put), #{type => Type, name => ConnectorName}); + Type = list_to_atom("influxdb_" ++ atom_to_list(Protocol)), + maps:merge(values(Protocol, put), #{type => Type, name => <<"connector">>}); values(udp, put) -> #{ host => <<"127.0.0.1">>, @@ -219,7 +223,7 @@ do_start_client(InstId, ClientConfig, Config = #{egress := #{payload := PayloadB end. client_config( - _InstId, + InstId, Config = #{ host := Host, port := Port, @@ -230,7 +234,7 @@ client_config( {host, Host}, {port, Port}, {pool_size, PoolSize}, - {pool, atom_pname_todo}, + {pool, binary_to_atom(InstId, utf8)}, {precision, maps:get(precision, Config, ms)} ] ++ protocol_config(Config). From 55b96845416a636fb831cbc28ca9d77ff04ed329 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Sun, 31 Jul 2022 20:37:13 +0800 Subject: [PATCH 12/71] fix: influxdb bridge api available --- apps/emqx_bridge/src/emqx_bridge_schema.erl | 45 +++-- .../i18n/emqx_ee_bridge_influxdb.conf | 40 +++- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 37 +++- .../src/emqx_ee_bridge_hstream.erl | 16 +- .../src/emqx_ee_bridge_influxdb.erl | 135 +++++++++----- .../src/emqx_ee_connector.app.src | 3 +- .../src/emqx_ee_connector_influxdb.erl | 172 +++++++++++++++--- 7 files changed, 336 insertions(+), 112 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 0b885db8c..46039390d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -36,24 +36,31 @@ post_request() -> http_schema("post"). http_schema(Method) -> - Schemas = lists:flatmap( - fun(Type) -> - [ - ref(schema_mod(Type), Method ++ "_ingress"), - ref(schema_mod(Type), Method ++ "_egress") - ] - end, - ?CONN_TYPES - ), - ExtSchemas = [ref(Module, Method) || Module <- schema_modules()], - hoconsc:union(Schemas ++ ExtSchemas). + Broker = + lists:flatmap( + fun(Type) -> + [ + ref(schema_mod(Type), Method ++ "_ingress"), + ref(schema_mod(Type), Method ++ "_egress") + ] + end, + ?CONN_TYPES + ) ++ [ref(Module, Method) || Module <- [emqx_bridge_webhook_schema]], + EE = ee_schemas(Method), + hoconsc:union(Broker ++ EE). -if(?EMQX_RELEASE_EDITION == ee). -schema_modules() -> - [emqx_bridge_webhook_schema] ++ emqx_ee_bridge:schema_modules(). +ee_schemas(Method) -> + emqx_ee_bridge:api_schemas(Method). + +ee_fields_bridges() -> + emqx_ee_bridge:fields(bridges). -else. -schema_modules() -> - [emqx_bridge_webhook_schema]. +ee_schemas(_) -> + []. + +ee_fields_bridges() -> + []. -endif. common_bridge_fields(ConnectorRef) -> @@ -158,14 +165,6 @@ fields("node_status") -> {"status", mk(status(), #{})} ]. --if(?EMQX_RELEASE_EDITION == ee). -ee_fields_bridges() -> - emqx_ee_bridge:fields(bridges). --else. -ee_fields_bridges() -> - []. --endif. - desc(bridges) -> ?DESC("desc_bridges"); desc("metrics") -> diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf index f421fb912..3930825e5 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf @@ -18,14 +18,44 @@ will be forwarded. zh: "本地 Topic" } } - payload { + measurement { desc { - en: """The payload to be forwarded to the InfluxDB. Placeholders supported.""" - zh: """要转发到 InfluxDB 的数据内容，支持占位符""" + en: """The measurement name to be forwarded to the InfluxDB. Placeholders supported.""" + zh: """要转发到 InfluxDB 的 Measurement 名称，支持占位符""" } label { - en: "Payload" - zh: "消息内容" + en: "Measurement" + zh: "Measurement" + } + } + timestamp { + desc { + en: """The timestamp to be forwarded to the InfluxDB. Placeholders supported. Default is message timestamp""" + zh: """要转发到 InfluxDB 的时间戳，支持占位符。默认使用消息的时间戳""" + } + label { + en: "Timestamp" + zh: "Timestamp" + } + } + tags { + desc { + en: """The tags to be forwarded to the InfluxDB. Placeholders supported.""" + zh: """要转发到 InfluxDB 的 Tags 数据内容，支持占位符""" + } + label { + en: "Tags" + zh: "Tags" + } + } + fields { + desc { + en: """The fields to be forwarded to the InfluxDB. Placeholders supported.""" + zh: """要转发到 InfluxDB 的 fields 数据内容，支持占位符""" + } + label { + en: "Fields" + zh: "Fields" } } config_enable { diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index ff9661cc8..881609bfc 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -6,26 +6,43 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -export([ - schema_modules/0, + api_schemas/1, conn_bridge_examples/1, resource_type/1, fields/1 ]). +api_schemas(Method) -> + [ + ref(emqx_ee_bridge_hstream, Method), + ref(emqx_ee_bridge_influxdb, Method ++ "_udp"), + ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"), + ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2") + ]. + schema_modules() -> - [emqx_ee_bridge_hstream, emqx_ee_bridge_influxdb]. + [ + emqx_ee_bridge_hstream, + emqx_ee_bridge_influxdb + ]. conn_bridge_examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, Fun = fun(Module, Examples) -> - Example = erlang:apply(Module, conn_bridge_example, [Method]), - maps:merge(Examples, Example) + ConnectorExamples = erlang:apply(Module, conn_bridge_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) end, lists:foldl(Fun, #{}, schema_modules()). resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); resource_type(hstreamdb) -> emqx_ee_connector_hstream; -resource_type(influxdb) -> emqx_ee_connector_influxdb. +resource_type(influxdb_udp) -> emqx_ee_connector_influxdb; +resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb; +resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb. fields(bridges) -> [ @@ -33,10 +50,14 @@ fields(bridges) -> mk( hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), #{desc => <<"EMQX Enterprise Config">>} - )}, - {influxdb, + )} + ] ++ fields(influxdb); +fields(influxdb) -> + [ + {Protocol, mk( - hoconsc:map(name, ref(emqx_ee_bridge_influxdb, "config")), + hoconsc:map(name, ref(emqx_ee_bridge_influxdb, Protocol)), #{desc => <<"EMQX Enterprise Config">>} )} + || Protocol <- [influxdb_udp, influxdb_api_v1, influxdb_api_v2] ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl index 73f7a20eb..200e695da 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl @@ -10,7 +10,7 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -export([ - conn_bridge_example/1 + conn_bridge_examples/1 ]). -export([ @@ -23,13 +23,15 @@ %% ------------------------------------------------------------------------------------------------- %% api -conn_bridge_example(Method) -> - #{ - <<"hstreamdb">> => #{ - summary => <<"HStreamDB Bridge">>, - value => values(Method) +conn_bridge_examples(Method) -> + [ + #{ + <<"hstreamdb">> => #{ + summary => <<"HStreamDB Bridge">>, + value => values(Method) + } } - }. + ]. values(get) -> maps:merge(values(post), ?METRICS_EXAMPLE); diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index dca55ba23..a19459208 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -10,7 +10,7 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -export([ - conn_bridge_example/1 + conn_bridge_examples/1 ]). -export([ @@ -23,28 +23,48 @@ %% ------------------------------------------------------------------------------------------------- %% api -conn_bridge_example(Method) -> - #{ - <<"influxdb">> => #{ - summary => <<"InfluxDB Bridge">>, - value => values(Method) +conn_bridge_examples(Method) -> + [ + #{ + <<"influxdb_udp">> => #{ + summary => <<"InfluxDB UDP Bridge">>, + value => values("influxdb_udp", Method) + } + }, + #{ + <<"influxdb_api_v1">> => #{ + summary => <<"InfluxDB HTTP API V1 Bridge">>, + value => values("influxdb_api_v1", Method) + } + }, + #{ + <<"influxdb_api_v2">> => #{ + summary => <<"InfluxDB HTTP API V2 Bridge">>, + value => values("influxdb_api_v2", Method) + } } - }. + ]. -values(get) -> - maps:merge(values(post), ?METRICS_EXAMPLE); -values(post) -> +values(Protocol, get) -> + maps:merge(values(Protocol, post), ?METRICS_EXAMPLE); +values(Protocol, post) -> #{ - type => influxdb, + type => list_to_atom(Protocol), name => <<"demo">>, - connector => <<"influxdb:api_v2_connector">>, + connector => list_to_binary(Protocol ++ ":connector"), enable => true, direction => egress, local_topic => <<"local/topic/#">>, - payload => <<"${payload}">> + measurement => <<"${topic}">>, + tags => #{<<"clientid">> => <<"${clientid}">>}, + fields => #{ + <<"payload">> => <<"${payload}">>, + <<"int_value">> => [int, <<"${payload.int_key}">>], + <<"uint_value">> => [uint, <<"${payload.uint_key}">>] + } }; -values(put) -> - values(post). +values(Protocol, put) -> + values(Protocol, post). %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions @@ -52,40 +72,69 @@ namespace() -> "bridge". roots() -> []. -fields("config") -> +fields(basic) -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, - {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})}, - {connector, field(connector)} + {measurement, mk(binary(), #{desc => ?DESC("measurement"), required => true})}, + {timestamp, + mk(binary(), #{ + desc => ?DESC("timestamp"), default => <<"${timestamp}">>, required => false + })}, + {tags, mk(map(), #{desc => ?DESC("tags"), required => false})}, + {fields, mk(map(), #{desc => ?DESC("fields"), required => true})} ]; -fields("post") -> - [ - {type, mk(enum([influxdb]), #{required => true, desc => ?DESC("desc_type")})}, - {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} - | fields("config") - ]; -fields("put") -> - fields("config"); -fields("get") -> - emqx_bridge_schema:metrics_status_fields() ++ fields("post"). +fields("post_udp") -> + method_fileds(post, influxdb_udp); +fields("post_api_v1") -> + method_fileds(post, influxdb_api_v1); +fields("post_api_v2") -> + method_fileds(post, influxdb_api_v2); +fields("put_udp") -> + method_fileds(put, influxdb_udp); +fields("put_api_v1") -> + method_fileds(put, influxdb_api_v1); +fields("put_api_v2") -> + method_fileds(put, influxdb_api_v2); +fields("get_udp") -> + method_fileds(get, influxdb_udp); +fields("get_api_v1") -> + method_fileds(get, influxdb_api_v1); +fields("get_api_v2") -> + method_fileds(get, influxdb_api_v2); +fields(Name) when + Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2 +-> + fields(basic) ++ connector_field(Name). -field(connector) -> - ConnectorConfigRef = - [ - ref(emqx_ee_connector_influxdb, influxdb_udp), - ref(emqx_ee_connector_influxdb, influxdb_api_v1), - ref(emqx_ee_connector_influxdb, influxdb_api_v2) - ], - mk( - hoconsc:union([binary() | ConnectorConfigRef]), - #{ - required => true, - example => <<"influxdb:demo">>, - desc => ?DESC("desc_connector") - } - ). +method_fileds(post, ConnectorType) -> + fields(basic) ++ connector_field(ConnectorType) ++ type_name_field(ConnectorType); +method_fileds(get, ConnectorType) -> + fields(basic) ++ + emqx_bridge_schema:metrics_status_fields() ++ + connector_field(ConnectorType) ++ type_name_field(ConnectorType); +method_fileds(put, ConnectorType) -> + fields(basic) ++ connector_field(ConnectorType). + +connector_field(Type) -> + [ + {connector, + mk( + hoconsc:union([binary(), ref(emqx_ee_connector_influxdb, Type)]), + #{ + required => true, + example => list_to_binary(atom_to_list(Type) ++ ":connector"), + desc => ?DESC(<<"desc_connector">>) + } + )} + ]. + +type_name_field(Type) -> + [ + {type, mk(Type, #{required => true, desc => ?DESC("desc_type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} + ]. desc("config") -> ?DESC("desc_config"); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 6d9fea3c9..675a934aa 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -4,7 +4,8 @@ {applications, [ kernel, stdlib, - hstreamdb_erl + hstreamdb_erl, + influxdb ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 37915d2e8..e343a64b0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -33,8 +33,8 @@ on_start(InstId, Config) -> on_stop(_InstId, #{client := Client}) -> influxdb:stop_client(Client). -on_query(_InstId, {send_message, _Data}, _AfterQuery, _State) -> - ok. +on_query(InstId, {send_message, Data}, AfterQuery, State) -> + do_query(InstId, {send_message, Data}, AfterQuery, State). on_get_status(_InstId, #{client := Client}) -> case influxdb:is_alive(Client) of @@ -184,18 +184,36 @@ start_client(InstId, Config) -> {error, R} end. -do_start_client(InstId, ClientConfig, Config = #{egress := #{payload := PayloadBin}}) -> +do_start_client( + InstId, + ClientConfig, + Config = #{ + egress := #{ + measurement := Measurement, + timestamp := Timestamp, + tags := Tags, + fields := Fields + } + } +) -> case influxdb:start_client(ClientConfig) of {ok, Client} -> case influxdb:is_alive(Client) of true -> - Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin), + State = #{ + client => Client, + measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement), + timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp), + tags => to_tags_config(Tags), + fields => to_fields_config(Fields) + }, ?SLOG(info, #{ msg => "starting influxdb connector success", connector => InstId, - client => Client + client => Client, + state => State }), - #{client => Client, payload => Payload}; + {ok, State}; false -> ?SLOG(error, #{ msg => "starting influxdb connector failed", @@ -231,21 +249,15 @@ client_config( } ) -> [ - {host, Host}, + {host, binary_to_list(Host)}, {port, Port}, {pool_size, PoolSize}, {pool, binary_to_atom(InstId, utf8)}, - {precision, maps:get(precision, Config, ms)} + {precision, atom_to_binary(maps:get(precision, Config, ms), utf8)} ] ++ protocol_config(Config). +%% api v2 config protocol_config(#{ - protocol := udp -}) -> - [ - {protocol, udp} - ]; -protocol_config(#{ - protocol := api_v1, username := Username, password := Password, database := DB, @@ -254,13 +266,12 @@ protocol_config(#{ [ {protocol, http}, {version, v1}, - {username, Username}, - {password, Password}, - {database, DB}, - {ssl, SSL} + {username, binary_to_list(Username)}, + {password, binary_to_list(Password)}, + {database, binary_to_list(DB)} ] ++ ssl_config(SSL); +%% api v1 config protocol_config(#{ - protocol := api_v2, bucket := Bucket, org := Org, token := Token, @@ -269,11 +280,15 @@ protocol_config(#{ [ {protocol, http}, {version, v2}, - {bucket, Bucket}, - {org, Org}, - {token, Token}, - {ssl, SSL} - ] ++ ssl_config(SSL). + {bucket, binary_to_list(Bucket)}, + {org, binary_to_list(Org)}, + {token, Token} + ] ++ ssl_config(SSL); +%% udp config +protocol_config(_) -> + [ + {protocol, udp} + ]. ssl_config(#{enable := false}) -> [ @@ -284,3 +299,110 @@ ssl_config(SSL = #{enable := true}) -> {https_enabled, true}, {transport, ssl} ] ++ maps:to_list(maps:remove(enable, SSL)). + +%% ------------------------------------------------------------------------------------------------- +%% Query + +do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client}) -> + case data_to_point(Data, State) of + {ok, Point} -> + case influxdb:write(Client, [Point]) of + ok -> + ?SLOG(debug, #{ + msg => "influxdb write point success", + connector => InstId, + point => Point + }), + emqx_resource:query_success(AfterQuery); + {error, Reason} -> + ?SLOG(error, #{ + msg => "influxdb write point failed", + connector => InstId, + reason => Reason + }), + emqx_resource:query_failed(AfterQuery) + end; + {error, Reason} -> + ?SLOG(error, #{ + msg => "influxdb trans point failed", + connector => InstId, + reason => Reason + }), + {error, Reason} + end. + +%% ------------------------------------------------------------------------------------------------- +%% Tags & Fields Config Trans + +to_tags_config(Tags) -> + maps:fold(fun to_maps_config/3, #{}, Tags). + +to_fields_config(Fields) -> + maps:fold(fun to_maps_config/3, #{}, Fields). + +to_maps_config(K, [IntType, V], Res) when IntType == <<"int">> orelse IntType == <<"uint">> -> + NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)), + NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)), + Res#{NK => {binary_to_atom(IntType, utf8), NV}}; +to_maps_config(K, V, Res) -> + NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)), + NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)), + Res#{NK => NV}. + +%% ------------------------------------------------------------------------------------------------- +%% Tags & Fields Data Trans +data_to_point( + Data, + #{ + measurement := Measurement, + timestamp := Timestamp, + tags := Tags, + fields := Fields + } +) -> + TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, + case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of + [TimestampInt] when is_integer(TimestampInt) -> + Point = #{ + measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data), + timestamp => TimestampInt, + tags => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags), + fields => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields) + }, + {ok, Point}; + BadTimestamp -> + {error, {bad_timestamp, BadTimestamp}} + end. + +maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse IntType == uint -> + TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, + NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions), + NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions), + case {NK, NV} of + {[undefined], _} -> + Res; + {_, [undefined]} -> + Res; + {_, [IntV]} when is_integer(IntV) -> + Res#{NK => {IntType, IntV}} + end; +maps_config_to_data(K, V, {Data, Res}) -> + TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, + NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions), + NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions), + case {NK, NV} of + {[undefined], _} -> + Res; + {_, [undefined]} -> + Res; + _ -> + Res#{bin(NK) => NV} + end. + +data_filter(undefined) -> undefined; +data_filter(Int) when is_integer(Int) -> Int; +data_filter(Number) when is_number(Number) -> Number; +data_filter(Bool) when is_boolean(Bool) -> Bool; +data_filter(Data) -> bin(Data). + +bin(Data) -> emqx_plugin_libs_rule:bin(Data). From c1542e7a77bf7fcbf89f4299faa945d3e94e20a7 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 1 Aug 2022 10:03:16 +0800 Subject: [PATCH 13/71] fix: influxdb connector bad encode message --- .../src/emqx_ee_connector_influxdb.erl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index e343a64b0..c6528ffd0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -363,11 +363,13 @@ data_to_point( TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of [TimestampInt] when is_integer(TimestampInt) -> + {_, EncodeTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags), + {_, EncodeFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields), Point = #{ measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data), timestamp => TimestampInt, - tags => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags), - fields => maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields) + tags => EncodeTags, + fields => EncodeFields }, {ok, Point}; BadTimestamp -> @@ -380,11 +382,11 @@ maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse Int NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions), case {NK, NV} of {[undefined], _} -> - Res; + {Data, Res}; {_, [undefined]} -> - Res; + {Data, Res}; {_, [IntV]} when is_integer(IntV) -> - Res#{NK => {IntType, IntV}} + {Data, Res#{NK => {IntType, IntV}}} end; maps_config_to_data(K, V, {Data, Res}) -> TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, @@ -392,11 +394,11 @@ maps_config_to_data(K, V, {Data, Res}) -> NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions), case {NK, NV} of {[undefined], _} -> - Res; + {Data, Res}; {_, [undefined]} -> - Res; + {Data, Res}; _ -> - Res#{bin(NK) => NV} + {Data, Res#{bin(NK) => NV}} end. data_filter(undefined) -> undefined; From 4663e354bf8cd394cbe00f43af8f28f4ee685196 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 1 Aug 2022 10:31:33 +0800 Subject: [PATCH 14/71] fix: update influxdb sdk version to 1.1.3. Adapted master deps version --- lib-ee/emqx_ee_connector/rebar.config | 2 +- mix.exs | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 0a3f6866b..485b0120d 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.2"}}} + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.3"}}} ]}. {shell, [ diff --git a/mix.exs b/mix.exs index 039172eb9..fa9593d07 100644 --- a/mix.exs +++ b/mix.exs @@ -89,15 +89,10 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.11.2", override: true, runtime: false}, -<<<<<<< HEAD - {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"} + {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, + {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.3", override: true} ] ++ umbrella_apps() ++ enterprise_apps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() -======= - {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, - {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.2"} - ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() ->>>>>>> fix: add influxdb udp api_v1 api_v2 connector end defp umbrella_apps() do From 78a44cfb874d0159073d5f60f43dc3dd3ad6af9e Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 1 Aug 2022 10:43:21 +0800 Subject: [PATCH 15/71] fix: override ehttpc & ecpool dep version --- mix.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index fa9593d07..77438d213 100644 --- a/mix.exs +++ b/mix.exs @@ -47,7 +47,7 @@ defmodule EMQXUmbrella.MixProject do {:lc, github: "emqx/lc", tag: "0.3.1"}, {:redbug, "2.0.7"}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, - {:ehttpc, github: "emqx/ehttpc", tag: "0.3.0"}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.3.0", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, @@ -56,7 +56,7 @@ defmodule EMQXUmbrella.MixProject do {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, - {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, + {:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true}, {:replayq, "0.3.4", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, {:emqtt, github: "emqx/emqtt", tag: "1.6.0", override: true}, From d778a71d3ba439b9442a8c2541aa61be7c271601 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 2 Aug 2022 16:45:47 +0800 Subject: [PATCH 16/71] chore: unifined lib-ee name style --- lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/.gitignore | 0 lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/README.md | 0 lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/rebar.config | 0 .../src/emqx_ee_conf.app.src} | 6 +++--- .../src/emqx_ee_conf_schema.erl} | 0 .../test/emqx_ee_conf_schema_SUITE.erl} | 0 .../test/emqx_ee_conf_schema_tests.erl} | 0 7 files changed, 3 insertions(+), 3 deletions(-) rename lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/.gitignore (100%) rename lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/README.md (100%) rename lib-ee/{emqx_enterprise_conf => emqx_ee_conf}/rebar.config (100%) rename lib-ee/{emqx_enterprise_conf/src/emqx_enterprise_conf.app.src => emqx_ee_conf/src/emqx_ee_conf.app.src} (53%) rename lib-ee/{emqx_enterprise_conf/src/emqx_enterprise_conf_schema.erl => emqx_ee_conf/src/emqx_ee_conf_schema.erl} (100%) rename lib-ee/{emqx_enterprise_conf/test/emqx_enterprise_conf_schema_SUITE.erl => emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl} (100%) rename lib-ee/{emqx_enterprise_conf/test/emqx_enterprise_conf_schema_tests.erl => emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl} (100%) diff --git a/lib-ee/emqx_enterprise_conf/.gitignore b/lib-ee/emqx_ee_conf/.gitignore similarity index 100% rename from lib-ee/emqx_enterprise_conf/.gitignore rename to lib-ee/emqx_ee_conf/.gitignore diff --git a/lib-ee/emqx_enterprise_conf/README.md b/lib-ee/emqx_ee_conf/README.md similarity index 100% rename from lib-ee/emqx_enterprise_conf/README.md rename to lib-ee/emqx_ee_conf/README.md diff --git a/lib-ee/emqx_enterprise_conf/rebar.config b/lib-ee/emqx_ee_conf/rebar.config similarity index 100% rename from lib-ee/emqx_enterprise_conf/rebar.config rename to lib-ee/emqx_ee_conf/rebar.config diff --git a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src similarity index 53% rename from lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src rename to lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index a25ce1aae..771fdcb27 100644 --- a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ -{application, emqx_enterprise_conf, [ - {description, "EMQX Enterprise configuration schema"}, - {vsn, "0.1.0"}, +{application, emqx_ee_conf, [ + {description, "EMQX Enterprise Edition configuration schema"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl similarity index 100% rename from lib-ee/emqx_enterprise_conf/src/emqx_enterprise_conf_schema.erl rename to lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl diff --git a/lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_SUITE.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl similarity index 100% rename from lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_SUITE.erl rename to lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl diff --git a/lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_tests.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl similarity index 100% rename from lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_tests.erl rename to lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl From 10b19acd359e258eb13fbd7a7087cadce887fdd0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 2 Aug 2022 16:58:21 +0800 Subject: [PATCH 17/71] chore: module and fun call rename --- build | 2 +- git-blame-ignore-revs | 2 +- lib-ee/emqx_ee_conf/README.md | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 2 +- .../emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl | 12 ++++++------ .../emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl | 4 ++-- lib-ee/emqx_license/test/emqx_license_SUITE.erl | 6 +++--- mix.exs | 4 ++-- rebar.config.erl | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/build b/build index 07c16b69e..f00cdab12 100755 --- a/build +++ b/build @@ -84,7 +84,7 @@ make_docs() { fi case $PROFILE in emqx-enterprise) - SCHEMA_MODULE='emqx_enterprise_conf_schema' + SCHEMA_MODULE='emqx_ee_conf_schema' ;; *) SCHEMA_MODULE='emqx_conf_schema' diff --git a/git-blame-ignore-revs b/git-blame-ignore-revs index b21b6a552..41c6e5e49 100644 --- a/git-blame-ignore-revs +++ b/git-blame-ignore-revs @@ -13,7 +13,7 @@ acb3544d4b112121b5d9414237d2af7860ccc2a3 # reformat lib-ee/emqx_license 4f396cceb84d79d5ef540e91c1a8420e8de74a56 4e3fd9febd0df11f3fe5f221cd2c4362be57c886 -# reformat lib-ee/emqx_enterprise_conf +# reformat lib-ee/emqx_ee_conf 1aa82992616ad848539a533a5cd20ba6f9071e5a # reformat apps/emqx_gateway 3f6d78dda03fd0d8e968a352e134f11a7f16bfe8 diff --git a/lib-ee/emqx_ee_conf/README.md b/lib-ee/emqx_ee_conf/README.md index b5b28dfdb..701d285cc 100644 --- a/lib-ee/emqx_ee_conf/README.md +++ b/lib-ee/emqx_ee_conf/README.md @@ -1,3 +1,3 @@ -# emqx_enterprise_conf +# emqx_ee_conf EMQX Enterprise configuration schema diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 771fdcb27..324e7e308 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.1"}, + {vsn, "0.1.0"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 60aeb1f81..38f6689c5 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_enterprise_conf_schema). +-module(emqx_ee_conf_schema). -behaviour(hocon_schema). diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl index 396faa4f5..0d6d4f061 100644 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl +++ b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_enterprise_conf_schema_SUITE). +-module(emqx_ee_conf_schema_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -20,12 +20,12 @@ all() -> t_namespace(_Config) -> ?assertEqual( emqx_conf_schema:namespace(), - emqx_enterprise_conf_schema:namespace() + emqx_ee_conf_schema:namespace() ). t_roots(_Config) -> BaseRoots = emqx_conf_schema:roots(), - EnterpriseRoots = emqx_enterprise_conf_schema:roots(), + EnterpriseRoots = emqx_ee_conf_schema:roots(), ?assertEqual([], BaseRoots -- EnterpriseRoots), @@ -42,12 +42,12 @@ t_roots(_Config) -> t_fields(_Config) -> ?assertEqual( emqx_conf_schema:fields("node"), - emqx_enterprise_conf_schema:fields("node") + emqx_ee_conf_schema:fields("node") ). t_translations(_Config) -> - [Root | _] = emqx_enterprise_conf_schema:translations(), + [Root | _] = emqx_ee_conf_schema:translations(), ?assertEqual( emqx_conf_schema:translation(Root), - emqx_enterprise_conf_schema:translation(Root) + emqx_ee_conf_schema:translation(Root) ). diff --git a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl index d7c4e35dd..b4bf0de3d 100644 --- a/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl +++ b/lib-ee/emqx_ee_conf/test/emqx_ee_conf_schema_tests.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_enterprise_conf_schema_tests). +-module(emqx_ee_conf_schema_tests). -include_lib("eunit/include/eunit.hrl"). @@ -22,7 +22,7 @@ doc_gen_test() -> "priv", "i18n.conf" ]), - _ = emqx_conf:dump_schema(Dir, emqx_enterprise_conf_schema, I18nFile), + _ = emqx_conf:dump_schema(Dir, emqx_ee_conf_schema, I18nFile), ok end }. diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index a648595d2..ef7c86f8e 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -104,10 +104,10 @@ setup_test(TestCase, Config) when [ {apps, [emqx_conf, emqx_license]}, {load_schema, false}, - {schema_mod, emqx_enterprise_conf_schema}, + {schema_mod, emqx_ee_conf_schema}, {env_handler, fun (emqx) -> - emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema), + emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema), %% emqx_config:save_schema_mod_and_names(emqx_license_schema), application:set_env(emqx, boot_modules, []), application:set_env( @@ -121,7 +121,7 @@ setup_test(TestCase, Config) when ), ok; (emqx_conf) -> - emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema), + emqx_config:save_schema_mod_and_names(emqx_ee_conf_schema), %% emqx_config:save_schema_mod_and_names(emqx_license_schema), application:set_env( emqx, diff --git a/mix.exs b/mix.exs index 77438d213..5209de843 100644 --- a/mix.exs +++ b/mix.exs @@ -236,7 +236,7 @@ defmodule EMQXUmbrella.MixProject do if(edition_type == :enterprise, do: [ emqx_license: :permanent, - emqx_enterprise_conf: :load, + emqx_ee_conf: :load, emqx_ee_connector: :permanent, emqx_ee_bridge: :permanent ], @@ -603,7 +603,7 @@ defmodule EMQXUmbrella.MixProject do end end - defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_conf_schema + defp emqx_schema_mod(:enterprise), do: :emqx_ee_conf_schema defp emqx_schema_mod(:community), do: :emqx_conf_schema defp bcrypt_dep() do diff --git a/rebar.config.erl b/rebar.config.erl index 96754f7f8..0a5208b61 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -280,7 +280,7 @@ overlay_vars_edition(ce) -> ]; overlay_vars_edition(ee) -> [ - {emqx_schema_mod, emqx_enterprise_conf_schema}, + {emqx_schema_mod, emqx_ee_conf_schema}, {is_enterprise, "yes"} ]. @@ -378,7 +378,7 @@ is_app(Name) -> relx_apps_per_edition(ee) -> [ emqx_license, - {emqx_enterprise_conf, load}, + {emqx_ee_conf, load}, emqx_ee_connector, emqx_ee_bridge ]; From 88fd7e14dc27dc7fd6dbde2fd32c879c2270b89a Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 28 Jul 2022 14:49:57 +0800 Subject: [PATCH 18/71] feat(bridge): add mysql sink --- .../i18n/emqx_ee_bridge_mysql.conf | 75 +++++++++++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 10 +- .../src/emqx_ee_bridge_mysql.erl | 104 ++++++++++++++++++ .../i18n/emqx_ee_connector_mysql.conf | 23 ++++ .../src/emqx_ee_connector_mysql.erl | 66 +++++++++++ 5 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf new file mode 100644 index 000000000..9e3796487 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf @@ -0,0 +1,75 @@ +emqx_ee_bridge_mysql { + sql { + desc { + en: """SQL Template""" + zh: """SQL 模板""" + } + label { + en: "SQL Template" + zh: "SQL 模板" + } + } + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用桥接""" + } + label { + en: "Enable Or Disable Bridge" + zh: "启用/禁用桥接" + } + } + config_direction { + desc { + en: """The direction of this bridge, MUST be 'egress'""" + zh: """桥接的方向， 必须是 egress""" + } + label { + en: "Bridge Direction" + zh: "桥接方向" + } + } + + desc_config { + desc { + en: """Configuration for an HStreamDB bridge.""" + zh: """HStreamDB 桥接配置""" + } + label: { + en: "HStreamDB Bridge Configuration" + zh: "HStreamDB 桥接配置" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: "Bridge Type" + zh: "桥接类型" + } + } + + desc_name { + desc { + en: """Bridge name, used as a human-readable description of the bridge.""" + zh: """桥接名字，可读描述""" + } + label { + en: "Bridge Name" + zh: "桥接名字" + } + } + desc_connector { + desc { + en: """Generic configuration for the connector.""" + zh: """连接器的通用配置。""" + } + label: { + en: "Connector Generic Configuration" + zh: "连接器通用配置。" + } + } +} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 881609bfc..5132ee249 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -15,6 +15,7 @@ api_schemas(Method) -> [ ref(emqx_ee_bridge_hstream, Method), + ref(emqx_ee_bridge_mysql, Method), ref(emqx_ee_bridge_influxdb, Method ++ "_udp"), ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"), ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2") @@ -23,7 +24,8 @@ api_schemas(Method) -> schema_modules() -> [ emqx_ee_bridge_hstream, - emqx_ee_bridge_influxdb + emqx_ee_bridge_influxdb, + emqx_ee_bridge_mysql ]. conn_bridge_examples(Method) -> @@ -40,6 +42,7 @@ conn_bridge_examples(Method) -> resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); resource_type(hstreamdb) -> emqx_ee_connector_hstream; +resource_type(mysql) -> emqx_ee_connector_mysql; resource_type(influxdb_udp) -> emqx_ee_connector_influxdb; resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb; resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb. @@ -50,6 +53,11 @@ fields(bridges) -> mk( hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), #{desc => <<"EMQX Enterprise Config">>} + )}, + {mysql, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_mysql, "config")), + #{desc => <<"EMQX Enterprise Config">>} )} ] ++ fields(influxdb); fields(influxdb) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl new file mode 100644 index 000000000..7d586d7eb --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -0,0 +1,104 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_mysql). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_ee_bridge.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_SQL, << + "insert into t_mqtt_msg(msgid, topic, qos, payload, arrived) " + "values (${id}, ${topic}, ${qos}, ${payload}, FROM_UNIXTIME(${timestamp}/1000))" +>>). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"mysql">> => #{ + summary => <<"MySQL Bridge">>, + value => values(Method) + } + } + ]. + +values(get) -> + maps:merge(values(post), ?METRICS_EXAMPLE); +values(post) -> + #{ + type => mysql, + name => <<"mysql">>, + connector => #{ + server => <<"127.0.0.1:3306">>, + database => <<"test">>, + pool_size => 8, + username => <<"root">>, + password => <<"public">>, + auto_reconnect => true + }, + enable => true, + direction => egress, + sql => ?DEFAULT_SQL + }; +values(put) -> + values(post). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, + {sql, mk(binary(), #{default => ?DEFAULT_SQL, desc => ?DESC("sql")})}, + {connector, field(connector)} + ]; +fields("post") -> + [type_field(), name_field() | fields("config")]; +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:metrics_status_fields() ++ fields("post"). + +field(connector) -> + mk( + ref(emqx_ee_connector_mysql, config), + #{ + required => true, + desc => ?DESC("desc_connector") + } + ). + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for MySQL using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- +%% internal +type_field() -> + {type, mk(enum([mysql]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf new file mode 100644 index 000000000..5d7c4848a --- /dev/null +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf @@ -0,0 +1,23 @@ +emqx_ee_connector_mysql { + type { + desc { + en: "The Connector Type." + zh: "连接器类型。" + } + label: { + en: "Connector Type" + zh: "连接器类型" + } + } + + name { + desc { + en: "Connector name, used as a human-readable description of the connector." + zh: "连接器名称，人类可读的连接器描述。" + } + label: { + en: "Connector Name" + zh: "连接器名称" + } + } +} diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl new file mode 100644 index 000000000..c4efff91c --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_connector_mysql). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-import(hoconsc, [mk/2, enum/1]). + +-behaviour(emqx_resource). + +%% callbacks of behaviour emqx_resource +-export([ + on_start/2, + on_stop/2, + on_query/4, + on_get_status/2 +]). + +-export([ + roots/0, + fields/1 +]). + +-define(SEND_MSG_KEY, send_message). + +%% ------------------------------------------------------------------------------------------------- +%% resource callback + +on_start(InstId, #{egress := #{sql := SQL}} = Config) -> + {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'), + {ok, State} = emqx_connector_mysql:on_start(InstId, Config#{ + prepare_statement => #{?SEND_MSG_KEY => PrepareSQL} + }), + {ok, State#{'ParamsTokens' => ParamsTokens}}. + +on_stop(InstId, State) -> + emqx_connector_mysql:on_stop(InstId, State). + +on_query( + InstId, + {?SEND_MSG_KEY, Msg}, + AfterQuery, + #{'ParamsTokens' := ParamsTokens} = State +) -> + Data = emqx_plugin_libs_rule:proc_sql(ParamsTokens, Msg), + emqx_connector_mysql:on_query( + InstId, {prepared_query, ?SEND_MSG_KEY, Data}, AfterQuery, State + ). + +on_get_status(InstId, State) -> + emqx_connector_mysql:on_get_status(InstId, State). + +%% ------------------------------------------------------------------------------------------------- +%% schema + +roots() -> + fields(config). + +fields(config) -> + emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields(). + +%% ------------------------------------------------------------------------------------------------- +%% internal functions From aa6077bdeaee1e08f89329b39a2f177adda69db7 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 4 Aug 2022 14:52:16 +0800 Subject: [PATCH 19/71] fix(bridge): remove emqx_ee_connector_mysql --- .../src/emqx_connector_mysql.erl | 62 +++++++++++++---- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 2 +- .../src/emqx_ee_bridge_mysql.erl | 42 +++++++----- .../i18n/emqx_ee_connector_mysql.conf | 23 ------- .../src/emqx_ee_connector_mysql.erl | 66 ------------------- 5 files changed, 78 insertions(+), 117 deletions(-) delete mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index d6963d04e..409da4060 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -44,6 +44,16 @@ default_port => ?MYSQL_DEFAULT_PORT }). +-type prepares() :: #{atom() => binary()}. +-type params_tokens() :: #{atom() => list()}. +-type state() :: + #{ + poolname := atom(), + prepare_statement := prepares(), + auto_reconnect := boolean(), + params_tokens := params_tokens() + }. + %%===================================================================== %% Hocon schema roots() -> @@ -63,6 +73,7 @@ server(desc) -> ?DESC("server"); server(_) -> undefined. %% =================================================================== +-spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. on_start( InstId, #{ @@ -97,8 +108,8 @@ on_start( {pool_size, PoolSize} ], PoolName = emqx_plugin_libs_pool:pool_name(InstId), - Prepares = maps:get(prepare_statement, Config, #{}), - State = #{poolname => PoolName, prepare_statement => Prepares, auto_reconnect => AutoReconn}, + Prepares = parse_prepare_sql(maps:get(prepare_statement, Config, #{})), + State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of ok -> {ok, init_prepare(State)}; {error, Reason} -> {error, Reason} @@ -111,13 +122,13 @@ on_stop(InstId, #{poolname := PoolName}) -> }), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {Type, SQLOrKey}, AfterQuery, State) -> - on_query(InstId, {Type, SQLOrKey, [], default_timeout}, AfterQuery, State); -on_query(InstId, {Type, SQLOrKey, Params}, AfterQuery, State) -> - on_query(InstId, {Type, SQLOrKey, Params, default_timeout}, AfterQuery, State); +on_query(InstId, {TypeOrKey, SQLOrKey}, AfterQuery, State) -> + on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, AfterQuery, State); +on_query(InstId, {TypeOrKey, SQLOrKey, Params}, AfterQuery, State) -> + on_query(InstId, {TypeOrKey, SQLOrKey, Params, default_timeout}, AfterQuery, State); on_query( InstId, - {Type, SQLOrKey, Params, Timeout}, + {TypeOrKey, SQLOrKey, Params, Timeout}, AfterQuery, #{poolname := PoolName, prepare_statement := Prepares} = State ) -> @@ -125,8 +136,9 @@ on_query( ?TRACE("QUERY", "mysql_connector_received", LogMeta), Worker = ecpool:get_client(PoolName), {ok, Conn} = ecpool_worker:client(Worker), - MySqlFunction = mysql_function(Type), - Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey, Params, Timeout]), + MySqlFunction = mysql_function(TypeOrKey), + {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State), + Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey2, Data, Timeout]), case Result of {error, disconnected} -> ?SLOG( @@ -145,7 +157,7 @@ on_query( case prepare_sql(Prepares, PoolName) of ok -> %% not return result, next loop will try again - on_query(InstId, {Type, SQLOrKey, Params, Timeout}, AfterQuery, State); + on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, AfterQuery, State); {error, Reason} -> ?SLOG( error, @@ -166,8 +178,13 @@ on_query( Result end. -mysql_function(sql) -> query; -mysql_function(prepared_query) -> execute. +mysql_function(sql) -> + query; +mysql_function(prepared_query) -> + execute; +%% for bridge +mysql_function(_) -> + mysql_function(prepared_query). on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) -> case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of @@ -287,3 +304,24 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> unprepare_sql_to_conn(Conn, PrepareSqlKey) -> mysql:unprepare(Conn, PrepareSqlKey). + +parse_prepare_sql(SQL) -> + parse_prepare_sql(maps:to_list(SQL), #{}, #{}). + +parse_prepare_sql([{Key, H} | T], SQL, Tokens) -> + {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H), + parse_prepare_sql(T, SQL#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}); +parse_prepare_sql([], SQL, Tokens) -> + #{prepare_statement => SQL, params_tokens => Tokens}. + +proc_sql_params(query, SQLOrKey, Params, _State) -> + {SQLOrKey, Params}; +proc_sql_params(prepared_query, SQLOrKey, Params, _State) -> + {SQLOrKey, Params}; +proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) -> + case maps:get(TypeOrKey, ParamsTokens, undefined) of + undefined -> + {SQLOrData, Params}; + Tokens -> + {TypeOrKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)} + end. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 5132ee249..a690b50d0 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -42,7 +42,7 @@ conn_bridge_examples(Method) -> resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); resource_type(hstreamdb) -> emqx_ee_connector_hstream; -resource_type(mysql) -> emqx_ee_connector_mysql; +resource_type(mysql) -> emqx_connector_mysql; resource_type(influxdb_udp) -> emqx_ee_connector_influxdb; resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb; resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index 7d586d7eb..bf38da2f8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -50,11 +50,11 @@ values(post) -> pool_size => 8, username => <<"root">>, password => <<"public">>, - auto_reconnect => true + auto_reconnect => true, + prepare_statement => #{send_message => ?DEFAULT_SQL} }, enable => true, - direction => egress, - sql => ?DEFAULT_SQL + direction => egress }; values(put) -> values(post). @@ -69,27 +69,29 @@ fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, - {sql, mk(binary(), #{default => ?DEFAULT_SQL, desc => ?DESC("sql")})}, - {connector, field(connector)} + {connector, + mk( + ref(?MODULE, connector), + #{ + required => true, + desc => ?DESC("desc_connector") + } + )} ]; fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> fields("config"); fields("get") -> - emqx_bridge_schema:metrics_status_fields() ++ fields("post"). - -field(connector) -> - mk( - ref(emqx_ee_connector_mysql, config), - #{ - required => true, - desc => ?DESC("desc_connector") - } - ). + emqx_bridge_schema:metrics_status_fields() ++ fields("post"); +fields(connector) -> + (emqx_connector_mysql:fields(config) -- + emqx_connector_schema_lib:prepare_statement_fields()) ++ prepare_statement_fields(). desc("config") -> ?DESC("desc_config"); +desc(connector) -> + ?DESC("desc_connector"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for MySQL using `", string:to_upper(Method), "` method."]; desc(_) -> @@ -102,3 +104,13 @@ type_field() -> name_field() -> {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. + +prepare_statement_fields() -> + [ + {prepare_statement, + mk(map(), #{ + desc => ?DESC(emqx_connector_schema_lib, prepare_statement), + default => #{<<"send_message">> => ?DEFAULT_SQL}, + example => #{<<"send_message">> => ?DEFAULT_SQL} + })} + ]. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf deleted file mode 100644 index 5d7c4848a..000000000 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_mysql.conf +++ /dev/null @@ -1,23 +0,0 @@ -emqx_ee_connector_mysql { - type { - desc { - en: "The Connector Type." - zh: "连接器类型。" - } - label: { - en: "Connector Type" - zh: "连接器类型" - } - } - - name { - desc { - en: "Connector name, used as a human-readable description of the connector." - zh: "连接器名称，人类可读的连接器描述。" - } - label: { - en: "Connector Name" - zh: "连接器名称" - } - } -} diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl deleted file mode 100644 index c4efff91c..000000000 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mysql.erl +++ /dev/null @@ -1,66 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_ee_connector_mysql). - --include_lib("hocon/include/hoconsc.hrl"). --include_lib("typerefl/include/types.hrl"). --include_lib("emqx/include/logger.hrl"). - --import(hoconsc, [mk/2, enum/1]). - --behaviour(emqx_resource). - -%% callbacks of behaviour emqx_resource --export([ - on_start/2, - on_stop/2, - on_query/4, - on_get_status/2 -]). - --export([ - roots/0, - fields/1 -]). - --define(SEND_MSG_KEY, send_message). - -%% ------------------------------------------------------------------------------------------------- -%% resource callback - -on_start(InstId, #{egress := #{sql := SQL}} = Config) -> - {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'), - {ok, State} = emqx_connector_mysql:on_start(InstId, Config#{ - prepare_statement => #{?SEND_MSG_KEY => PrepareSQL} - }), - {ok, State#{'ParamsTokens' => ParamsTokens}}. - -on_stop(InstId, State) -> - emqx_connector_mysql:on_stop(InstId, State). - -on_query( - InstId, - {?SEND_MSG_KEY, Msg}, - AfterQuery, - #{'ParamsTokens' := ParamsTokens} = State -) -> - Data = emqx_plugin_libs_rule:proc_sql(ParamsTokens, Msg), - emqx_connector_mysql:on_query( - InstId, {prepared_query, ?SEND_MSG_KEY, Data}, AfterQuery, State - ). - -on_get_status(InstId, State) -> - emqx_connector_mysql:on_get_status(InstId, State). - -%% ------------------------------------------------------------------------------------------------- -%% schema - -roots() -> - fields(config). - -fields(config) -> - emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields(). - -%% ------------------------------------------------------------------------------------------------- -%% internal functions From 4d0516c6e9bd3d2e1ba156e78ac096c58a2af581 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 4 Aug 2022 10:10:38 +0800 Subject: [PATCH 20/71] chore: use HStreamDB for module name --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++---- ...ee_bridge_hstream.erl => emqx_ee_bridge_hstreamdb.erl} | 4 ++-- ...ge_hstream_SUITE.erl => ee_bridge_hstreamdb_SUITE.erl} | 2 +- .../emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf | 2 +- lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl | 6 +++--- ...nector_hstream.erl => emqx_ee_connector_hstreamdb.erl} | 2 +- ...hstream_SUITE.erl => ee_connector_hstreamdb_SUITE.erl} | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) rename lib-ee/emqx_ee_bridge/src/{emqx_ee_bridge_hstream.erl => emqx_ee_bridge_hstreamdb.erl} (95%) rename lib-ee/emqx_ee_bridge/test/{ee_bridge_hstream_SUITE.erl => ee_bridge_hstreamdb_SUITE.erl} (92%) rename lib-ee/emqx_ee_connector/src/{emqx_ee_connector_hstream.erl => emqx_ee_connector_hstreamdb.erl} (99%) rename lib-ee/emqx_ee_connector/test/{ee_connector_hstream_SUITE.erl => ee_connector_hstreamdb_SUITE.erl} (91%) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf index 2e4397a04..dd3346579 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf @@ -1,4 +1,4 @@ -emqx_ee_bridge_hstream { +emqx_ee_bridge_hstreamdb { local_topic { desc { en: """ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index a690b50d0..ebc06d211 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -14,8 +14,8 @@ api_schemas(Method) -> [ - ref(emqx_ee_bridge_hstream, Method), ref(emqx_ee_bridge_mysql, Method), + ref(emqx_ee_bridge_hstreamdb, Method), ref(emqx_ee_bridge_influxdb, Method ++ "_udp"), ref(emqx_ee_bridge_influxdb, Method ++ "_api_v1"), ref(emqx_ee_bridge_influxdb, Method ++ "_api_v2") @@ -23,7 +23,7 @@ api_schemas(Method) -> schema_modules() -> [ - emqx_ee_bridge_hstream, + emqx_ee_bridge_hstreamdb, emqx_ee_bridge_influxdb, emqx_ee_bridge_mysql ]. @@ -41,7 +41,7 @@ conn_bridge_examples(Method) -> lists:foldl(Fun, #{}, schema_modules()). resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(hstreamdb) -> emqx_ee_connector_hstream; +resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(mysql) -> emqx_connector_mysql; resource_type(influxdb_udp) -> emqx_ee_connector_influxdb; resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb; @@ -51,7 +51,7 @@ fields(bridges) -> [ {hstreamdb, mk( - hoconsc:map(name, ref(emqx_ee_bridge_hstream, "config")), + hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")), #{desc => <<"EMQX Enterprise Config">>} )}, {mysql, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl similarity index 95% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl rename to lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl index 200e695da..79a50f35f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstream.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_hstream). +-module(emqx_ee_bridge_hstreamdb). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -71,7 +71,7 @@ fields("get") -> field(connector) -> mk( - hoconsc:union([binary(), ref(emqx_ee_connector_hstream, config)]), + hoconsc:union([binary(), ref(emqx_ee_connector_hstreamdb, config)]), #{ required => true, example => <<"hstreamdb:demo">>, diff --git a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl similarity index 92% rename from lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl rename to lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl index d03c13cac..429323ad7 100644 --- a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstream_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(ee_bridge_hstream_SUITE). +-module(ee_bridge_hstreamdb_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf index 080076065..dd9659aba 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf @@ -1,5 +1,5 @@ -emqx_ee_connector_hstream { +emqx_ee_connector_hstreamdb { type { desc { en: "The Connector Type." diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl index 9e8e7bac1..0971e1a06 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl @@ -13,7 +13,7 @@ api_schemas(Method) -> [ - ref(emqx_ee_connector_hstream, Method), + ref(emqx_ee_connector_hstreamdb, Method), ref(emqx_ee_connector_influxdb, Method ++ "_udp"), ref(emqx_ee_connector_influxdb, Method ++ "_api_v1"), ref(emqx_ee_connector_influxdb, Method ++ "_api_v2") @@ -23,7 +23,7 @@ fields(connectors) -> [ {hstreamdb, mk( - hoconsc:map(name, ref(emqx_ee_connector_hstream, config)), + hoconsc:map(name, ref(emqx_ee_connector_hstreamdb, config)), #{desc => <<"EMQX Enterprise Config">>} )} ] ++ fields(influxdb); @@ -52,6 +52,6 @@ connector_examples(Method) -> schema_modules() -> [ - emqx_ee_connector_hstream, + emqx_ee_connector_hstreamdb, emqx_ee_connector_influxdb ]. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl rename to lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl index 7a8ce3fe9..56e6e536c 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstream.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_hstream). +-module(emqx_ee_connector_hstreamdb). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl similarity index 91% rename from lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl rename to lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl index cebe77c6a..4de456b2b 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_hstream_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(ee_connector_hstream_SUITE). +-module(ee_connector_hstreamdb_SUITE). -compile(nowarn_export_all). -compile(export_all). From 2e336fbc018e44e4433d85893fa0f0bf0f49d457 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 4 Aug 2022 10:17:09 +0800 Subject: [PATCH 21/71] fix: bridge and connector namespace contains DB name Contains Detailed DB name for identifier hocon schema refs. --- apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 2 +- lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl | 2 ++ lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 2 ++ 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl index f11247d68..d833e6ca8 100644 --- a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl @@ -9,7 +9,7 @@ %%====================================================================================== %% Hocon Schema Definitions -namespace() -> "bridge". +namespace() -> "bridge_webhook". roots() -> []. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl index 79a50f35f..3b5183150 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl @@ -50,7 +50,7 @@ values(put) -> %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions -namespace() -> "bridge". +namespace() -> "bridge_hstreamdb". roots() -> []. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index a19459208..a55d9d47a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -68,7 +68,7 @@ values(Protocol, put) -> %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions -namespace() -> "bridge". +namespace() -> "bridge_influxdb". roots() -> []. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl index 56e6e536c..d0bbed136 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl @@ -24,6 +24,7 @@ ]). -export([ + namespace/0, roots/0, fields/1, connector_examples/1 @@ -75,6 +76,7 @@ on_flush_result({{flush, _Stream, _Records}, {error, _Reason}}) -> %% ------------------------------------------------------------------------------------------------- %% schema +namespace() -> connector_hstreamdb. roots() -> fields(config). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index c6528ffd0..8ba8478b3 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -20,6 +20,7 @@ ]). -export([ + namespace/0, fields/1, connector_examples/1 ]). @@ -46,6 +47,7 @@ on_get_status(_InstId, #{client := Client}) -> %% ------------------------------------------------------------------------------------------------- %% schema +namespace() -> connector_influxdb. fields("put_udp") -> fields(influxdb_udp); From 33a604dcaa1d0b3d4e50ee4a6b8def72ecc399e6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 4 Aug 2022 13:40:00 +0800 Subject: [PATCH 22/71] fix: refine influxdb connector api type name --- .../src/emqx_ee_connector.erl | 6 +-- .../src/emqx_ee_connector_influxdb.erl | 42 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl index 0971e1a06..6846ea740 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.erl @@ -14,9 +14,9 @@ api_schemas(Method) -> [ ref(emqx_ee_connector_hstreamdb, Method), - ref(emqx_ee_connector_influxdb, Method ++ "_udp"), - ref(emqx_ee_connector_influxdb, Method ++ "_api_v1"), - ref(emqx_ee_connector_influxdb, Method ++ "_api_v2") + ref(emqx_ee_connector_influxdb, "udp_" ++ Method), + ref(emqx_ee_connector_influxdb, "api_v1_" ++ Method), + ref(emqx_ee_connector_influxdb, "api_v2_" ++ Method) ]. fields(connectors) -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 8ba8478b3..c9971607e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -49,30 +49,30 @@ on_get_status(_InstId, #{client := Client}) -> %% schema namespace() -> connector_influxdb. -fields("put_udp") -> +fields("udp_get") -> + Key = influxdb_udp, + fields(Key) ++ type_name_field(Key); +fields("udp_post") -> + Key = influxdb_udp, + fields(Key) ++ type_name_field(Key); +fields("udp_put") -> fields(influxdb_udp); -fields("put_api_v1") -> +fields("api_v1_get") -> + Key = influxdb_api_v1, + fields(Key) ++ type_name_field(Key); +fields("api_v1_post") -> + Key = influxdb_api_v1, + fields(Key) ++ type_name_field(Key); +fields("api_v1_put") -> fields(influxdb_api_v1); -fields("put_api_v2") -> +fields("api_v2_get") -> + Key = influxdb_api_v2, + fields(Key) ++ type_name_field(Key); +fields("api_v2_post") -> + Key = influxdb_api_v2, + fields(Key) ++ type_name_field(Key); +fields("api_v2_put") -> fields(influxdb_api_v2); -fields("get_udp") -> - Key = influxdb_udp, - fields(Key) ++ type_name_field(Key); -fields("get_api_v1") -> - Key = influxdb_api_v1, - fields(Key) ++ type_name_field(Key); -fields("get_api_v2") -> - Key = influxdb_api_v2, - fields(Key) ++ type_name_field(Key); -fields("post_udp") -> - Key = influxdb_udp, - fields(Key) ++ type_name_field(Key); -fields("post_api_v1") -> - Key = influxdb_api_v1, - fields(Key) ++ type_name_field(Key); -fields("post_api_v2") -> - Key = influxdb_api_v2, - fields(Key) ++ type_name_field(Key); fields(basic) -> [ {host, From 06f246a8960fe8c2988604552e7f436da87a18f0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 8 Aug 2022 10:38:52 +0800 Subject: [PATCH 23/71] fix: prevent unexcepted square brackets in influxdb line protocol --- .../src/emqx_ee_connector_influxdb.erl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index c9971607e..f0e5cd333 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -379,9 +379,10 @@ data_to_point( end. maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse IntType == uint -> - TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, - NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions), - NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions), + KTransOptions = #{return => full_binary}, + VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, + NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions), + NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions), case {NK, NV} of {[undefined], _} -> {Data, Res}; @@ -391,16 +392,17 @@ maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse Int {Data, Res#{NK => {IntType, IntV}}} end; maps_config_to_data(K, V, {Data, Res}) -> - TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, - NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, TransOptions), - NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, TransOptions), + KTransOptions = #{return => full_binary}, + VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, + NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions), + NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions), case {NK, NV} of {[undefined], _} -> {Data, Res}; {_, [undefined]} -> {Data, Res}; _ -> - {Data, Res#{bin(NK) => NV}} + {Data, Res#{NK => NV}} end. data_filter(undefined) -> undefined; From 35e347aec82cc9ad2ed15dba6972f27406488ac6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 8 Aug 2022 17:48:39 +0800 Subject: [PATCH 24/71] feat: refine influxdb bridge conf Consistent influxdb line protocol config to raw syntax format. See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) --- .../i18n/emqx_ee_bridge_influxdb.conf | 56 +++---- .../src/emqx_ee_bridge_influxdb.erl | 99 +++++++++-- .../src/emqx_ee_connector_influxdb.erl | 157 +++++++++++------- 3 files changed, 203 insertions(+), 109 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf index 3930825e5..701608721 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf @@ -18,45 +18,31 @@ will be forwarded. zh: "本地 Topic" } } - measurement { + write_syntax { desc { - en: """The measurement name to be forwarded to the InfluxDB. Placeholders supported.""" - zh: """要转发到 InfluxDB 的 Measurement 名称，支持占位符""" + en: """ +Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeHolder supported. +See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and +[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
+TLDR: +``` +[,=[,=]] =[,=] [] +``` +""" + zh: """ +使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符
+参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及 +[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
+TLDR: +``` +[,=[,=]] =[,=] [] +``` +""" } label { - en: "Measurement" - zh: "Measurement" - } - } - timestamp { - desc { - en: """The timestamp to be forwarded to the InfluxDB. Placeholders supported. Default is message timestamp""" - zh: """要转发到 InfluxDB 的时间戳，支持占位符。默认使用消息的时间戳""" + en: "write_syntax" + zh: "写语句" } - label { - en: "Timestamp" - zh: "Timestamp" - } - } - tags { - desc { - en: """The tags to be forwarded to the InfluxDB. Placeholders supported.""" - zh: """要转发到 InfluxDB 的 Tags 数据内容，支持占位符""" - } - label { - en: "Tags" - zh: "Tags" - } - } - fields { - desc { - en: """The fields to be forwarded to the InfluxDB. Placeholders supported.""" - zh: """要转发到 InfluxDB 的 fields 数据内容，支持占位符""" - } - label { - en: "Fields" - zh: "Fields" - } } config_enable { desc { diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index a55d9d47a..3a6787af6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -3,9 +3,10 @@ %%-------------------------------------------------------------------- -module(emqx_ee_bridge_influxdb). +-include("emqx_ee_bridge.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --include("emqx_ee_bridge.hrl"). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -55,13 +56,10 @@ values(Protocol, post) -> enable => true, direction => egress, local_topic => <<"local/topic/#">>, - measurement => <<"${topic}">>, - tags => #{<<"clientid">> => <<"${clientid}">>}, - fields => #{ - <<"payload">> => <<"${payload}">>, - <<"int_value">> => [int, <<"${payload.int_key}">>], - <<"uint_value">> => [uint, <<"${payload.uint_key}">>] - } + write_syntax => + <<"${topic},clientid=${clientid}", " ", "payload=${payload},", + "${clientid}_int_value=${payload.int_key}i,", "uint_value=${payload.uint_key}u,", + "bool=${payload.bool}">> }; values(Protocol, put) -> values(Protocol, post). @@ -77,13 +75,7 @@ fields(basic) -> {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, - {measurement, mk(binary(), #{desc => ?DESC("measurement"), required => true})}, - {timestamp, - mk(binary(), #{ - desc => ?DESC("timestamp"), default => <<"${timestamp}">>, required => false - })}, - {tags, mk(map(), #{desc => ?DESC("tags"), required => false})}, - {fields, mk(map(), #{desc => ?DESC("fields"), required => true})} + {write_syntax, fun write_syntax/1} ]; fields("post_udp") -> method_fileds(post, influxdb_udp); @@ -142,3 +134,80 @@ desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for HStream using `", string:to_upper(Method), "` method."]; desc(_) -> undefined. + +write_syntax(type) -> + list(); +write_syntax(required) -> + true; +write_syntax(validator) -> + [?NOT_EMPTY("the value of the field 'write_syntax' cannot be empty")]; +write_syntax(converter) -> + fun converter_influx_lines/1; +write_syntax(desc) -> + ?DESC("write_syntax"); +write_syntax(_) -> + undefined. + +converter_influx_lines(RawLines) -> + Lines = string:tokens(str(RawLines), "\n"), + lists:reverse(lists:foldl(fun converter_influx_line/2, [], Lines)). + +converter_influx_line(Line, AccIn) -> + case string:tokens(str(Line), " ") of + [MeasurementAndTags, Fields, Timestamp] -> + {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags), + [ + #{ + measurement => Measurement, + tags => kv_pairs(Tags), + fields => kv_pairs(string:tokens(Fields, ",")), + timestamp => Timestamp + } + | AccIn + ]; + [MeasurementAndTags, Fields] -> + {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags), + %% TODO: fix here both here and influxdb driver. + %% Default value should evaluated by InfluxDB. + [ + #{ + measurement => Measurement, + tags => kv_pairs(Tags), + fields => kv_pairs(string:tokens(Fields, ",")), + timestamp => "${timestamp}" + } + | AccIn + ]; + _ -> + throw("Bad InfluxDB Line Protocol schema") + end. + +split_measurement_and_tags(Subject) -> + case string:tokens(Subject, ",") of + [] -> + throw("Bad Measurement schema"); + [Measurement] -> + {Measurement, []}; + [Measurement | Tags] -> + {Measurement, Tags} + end. + +kv_pairs(Pairs) -> + kv_pairs(Pairs, []). +kv_pairs([], Acc) -> + lists:reverse(Acc); +kv_pairs([Pair | Rest], Acc) -> + case string:tokens(Pair, "=") of + [K, V] -> + %% Reduplicated keys will be overwritten. Follows InfluxDB Line Protocol. + kv_pairs(Rest, [{K, V} | Acc]); + _ -> + throw(io_lib:format("Bad InfluxDB Line Protocol Key Value pair: ~p", Pair)) + end. + +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index f0e5cd333..aca92e791 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -191,10 +191,7 @@ do_start_client( ClientConfig, Config = #{ egress := #{ - measurement := Measurement, - timestamp := Timestamp, - tags := Tags, - fields := Fields + write_syntax := Lines } } ) -> @@ -204,10 +201,7 @@ do_start_client( true -> State = #{ client => Client, - measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement), - timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp), - tags => to_tags_config(Tags), - fields => to_fields_config(Fields) + write_syntax => to_config(Lines) }, ?SLOG(info, #{ msg => "starting influxdb connector success", @@ -306,46 +300,65 @@ ssl_config(SSL = #{enable := true}) -> %% Query do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client}) -> - case data_to_point(Data, State) of - {ok, Point} -> - case influxdb:write(Client, [Point]) of - ok -> - ?SLOG(debug, #{ - msg => "influxdb write point success", - connector => InstId, - point => Point - }), - emqx_resource:query_success(AfterQuery); - {error, Reason} -> - ?SLOG(error, #{ - msg => "influxdb write point failed", - connector => InstId, - reason => Reason - }), - emqx_resource:query_failed(AfterQuery) - end; - {error, Reason} -> + {Points, Errs} = data_to_points(Data, State), + lists:foreach( + fun({error, Reason}) -> ?SLOG(error, #{ msg => "influxdb trans point failed", connector => InstId, reason => Reason + }) + end, + Errs + ), + case influxdb:write(Client, Points) of + ok -> + ?SLOG(debug, #{ + msg => "influxdb write point success", + connector => InstId, + points => Points }), - {error, Reason} + emqx_resource:query_success(AfterQuery); + {error, Reason} -> + ?SLOG(error, #{ + msg => "influxdb write point failed", + connector => InstId, + reason => Reason + }), + emqx_resource:query_failed(AfterQuery) end. %% ------------------------------------------------------------------------------------------------- %% Tags & Fields Config Trans -to_tags_config(Tags) -> - maps:fold(fun to_maps_config/3, #{}, Tags). +to_config(Lines) -> + to_config(Lines, []). -to_fields_config(Fields) -> - maps:fold(fun to_maps_config/3, #{}, Fields). +to_config([], Acc) -> + lists:reverse(Acc); +to_config( + [ + #{ + measurement := Measurement, + timestamp := Timestamp, + tags := Tags, + fields := Fields + } + | Rest + ], + Acc +) -> + Res = #{ + measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement), + timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp), + tags => to_kv_config(Tags), + fields => to_kv_config(Fields) + }, + to_config(Rest, [Res | Acc]). + +to_kv_config(KVfields) -> + maps:fold(fun to_maps_config/3, #{}, proplists:to_map(KVfields)). -to_maps_config(K, [IntType, V], Res) when IntType == <<"int">> orelse IntType == <<"uint">> -> - NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)), - NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)), - Res#{NK => {binary_to_atom(IntType, utf8), NV}}; to_maps_config(K, V, Res) -> NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)), NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)), @@ -353,14 +366,24 @@ to_maps_config(K, V, Res) -> %% ------------------------------------------------------------------------------------------------- %% Tags & Fields Data Trans -data_to_point( +data_to_points(Data, #{write_syntax := Lines}) -> + lines_to_points(Data, Lines, [], []). + +lines_to_points(_, [], Points, Errs) -> + {Points, Errs}; +lines_to_points( Data, - #{ - measurement := Measurement, - timestamp := Timestamp, - tags := Tags, - fields := Fields - } + [ + #{ + measurement := Measurement, + timestamp := Timestamp, + tags := Tags, + fields := Fields + } + | Rest + ], + ResAcc, + ErrAcc ) -> TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of @@ -373,24 +396,11 @@ data_to_point( tags => EncodeTags, fields => EncodeFields }, - {ok, Point}; + lines_to_points(Data, Rest, [Point | ResAcc], ErrAcc); BadTimestamp -> - {error, {bad_timestamp, BadTimestamp}} + lines_to_points(Data, Rest, ResAcc, [{error, {bad_timestamp, BadTimestamp}} | ErrAcc]) end. -maps_config_to_data(K, {IntType, V}, {Data, Res}) when IntType == int orelse IntType == uint -> - KTransOptions = #{return => full_binary}, - VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, - NK = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions), - NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions), - case {NK, NV} of - {[undefined], _} -> - {Data, Res}; - {_, [undefined]} -> - {Data, Res}; - {_, [IntV]} when is_integer(IntV) -> - {Data, Res#{NK => {IntType, IntV}}} - end; maps_config_to_data(K, V, {Data, Res}) -> KTransOptions = #{return => full_binary}, VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, @@ -402,9 +412,38 @@ maps_config_to_data(K, V, {Data, Res}) -> {_, [undefined]} -> {Data, Res}; _ -> - {Data, Res#{NK => NV}} + {Data, Res#{NK => value_type(NV)}} end. +value_type([Int, <<"i">>]) when + is_integer(Int) +-> + {int, Int}; +value_type([UInt, <<"u">>]) -> + {uint, UInt}; +value_type([<<"t">>]) -> + 't'; +value_type([<<"T">>]) -> + 'T'; +value_type([<<"true">>]) -> + 'true'; +value_type([<<"TRUE">>]) -> + 'TRUE'; +value_type([<<"True">>]) -> + 'True'; +value_type([<<"f">>]) -> + 'f'; +value_type([<<"F">>]) -> + 'F'; +value_type([<<"false">>]) -> + 'false'; +value_type([<<"FALSE">>]) -> + 'FALSE'; +value_type([<<"False">>]) -> + 'False'; +value_type(Val) -> + Val. + data_filter(undefined) -> undefined; data_filter(Int) when is_integer(Int) -> Int; data_filter(Number) when is_number(Number) -> Number; From 7305c3c04c89996abdc74bc96979adaa72dee52f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 8 Aug 2022 18:22:46 +0800 Subject: [PATCH 25/71] fix(influxdb): parsed boolean type match --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index aca92e791..4a613e124 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -425,7 +425,7 @@ value_type([<<"t">>]) -> 't'; value_type([<<"T">>]) -> 'T'; -value_type([<<"true">>]) -> +value_type([true]) -> 'true'; value_type([<<"TRUE">>]) -> 'TRUE'; @@ -435,7 +435,7 @@ value_type([<<"f">>]) -> 'f'; value_type([<<"F">>]) -> 'F'; -value_type([<<"false">>]) -> +value_type([false]) -> 'false'; value_type([<<"FALSE">>]) -> 'FALSE'; From d51ca2672ac470dbd5fcc53cbf61ab233d505315 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 9 Aug 2022 09:49:14 +0800 Subject: [PATCH 26/71] fix: spellcheck and update checker repo version --- .github/workflows/build_slim_packages.yaml | 2 +- ...tream.conf => emqx_ee_bridge_hstreamdb.conf} | 0 .../i18n/emqx_ee_bridge_influxdb.conf | 3 ++- .../src/emqx_ee_bridge_influxdb.erl | 8 +++++++- ...am.conf => emqx_ee_connector_hstreamdb.conf} | 12 +++++++++++- .../i18n/emqx_ee_connector_influxdb.conf | 17 ++++++++--------- .../src/emqx_ee_connector_hstreamdb.erl | 4 ++++ .../src/emqx_ee_connector_influxdb.erl | 9 +++++++++ scripts/spellcheck | 2 +- 9 files changed, 43 insertions(+), 14 deletions(-) rename lib-ee/emqx_ee_bridge/i18n/{emqx_ee_bridge_hstream.conf => emqx_ee_bridge_hstreamdb.conf} (100%) rename lib-ee/emqx_ee_connector/i18n/{emqx_ee_connector_hstream.conf => emqx_ee_connector_hstreamdb.conf} (87%) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 27f43d0ea..488189d81 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -205,7 +205,7 @@ jobs: - emqx - emqx-enterprise runs-on: aws-amd64 - container: "ghcr.io/emqx/emqx-schema-validate:0.3.3" + container: "ghcr.io/emqx/emqx-schema-validate:0.3.5" steps: - uses: actions/download-artifact@v2 name: Download schema dump diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstream.conf rename to lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf index 701608721..ffd0b66a0 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf @@ -21,7 +21,7 @@ will be forwarded. write_syntax { desc { en: """ -Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeHolder supported. +Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
TLDR: @@ -97,6 +97,7 @@ TLDR: zh: "桥接名字" } } + desc_connector { desc { en: """Generic configuration for the connector.""" diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 3a6787af6..490abfc09 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -131,7 +131,13 @@ type_name_field(Type) -> desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> - ["Configuration for HStream using `", string:to_upper(Method), "` method."]; + ["Configuration for InfluxDB using `", string:to_upper(Method), "` method."]; +desc(influxdb_udp) -> + ?DESC(emqx_ee_connector_influxdb, "influxdb_udp"); +desc(influxdb_api_v1) -> + ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v1"); +desc(influxdb_api_v2) -> + ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v2"); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf similarity index 87% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf rename to lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf index dd9659aba..0826c8f0c 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstream.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf @@ -1,5 +1,15 @@ - emqx_ee_connector_hstreamdb { + config { + desc { + en: "HStreamDB connection config" + zh: "HStreamDB 连接配置。" + } + label: { + en: "Connection config" + zh: "连接配置" + } + } + type { desc { en: "The Connector Type." diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf index a909c9a72..7e223b9b7 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf @@ -1,4 +1,3 @@ - emqx_ee_connector_influxdb { type { desc { @@ -43,7 +42,7 @@ emqx_ee_connector_influxdb { } protocol { desc { - en: """InfluxDB protocol. UDP or HTTP API or HTTP API V2""" + en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2""" zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2""" } label: { @@ -51,9 +50,9 @@ emqx_ee_connector_influxdb { zh: """协议""" } } - protocol_udp { + influxdb_udp { desc { - en: """InfluxDB protocol.""" + en: """InfluxDB's UDP protocol.""" zh: """InfluxDB UDP 协议""" } label: { @@ -61,9 +60,9 @@ emqx_ee_connector_influxdb { zh: """UDP 协议""" } } - protocol_api_v1 { + influxdb_api_v1 { desc { - en: """InfluxDB protocol. Support InfluxDB v1.8 and before.""" + en: """InfluxDB's protocol. Support InfluxDB v1.8 and before.""" zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本""" } label: { @@ -71,9 +70,9 @@ emqx_ee_connector_influxdb { zh: """HTTP API 协议""" } } - protocol_api_v2 { + influxdb_api_v2 { desc { - en: """InfluxDB protocol. Support InfluxDB v2.0 and after.""" + en: """InfluxDB's protocol. Support InfluxDB v2.0 and after.""" zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本""" } label: { @@ -123,7 +122,7 @@ emqx_ee_connector_influxdb { } org { desc { - en: """InfluxDB organization name.""" + en: """Organization name of InfluxDB.""" zh: """InfluxDB 组织名称。""" } label: { diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl index d0bbed136..8ee37cd8a 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl @@ -27,6 +27,7 @@ namespace/0, roots/0, fields/1, + desc/1, connector_examples/1 ]). @@ -123,6 +124,9 @@ values(put) -> values(_) -> #{}. +desc(config) -> + ?DESC("config"). + %% ------------------------------------------------------------------------------------------------- %% internal functions start_client(InstId, Config) -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 4a613e124..9582f1729 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -22,6 +22,7 @@ -export([ namespace/0, fields/1, + desc/1, connector_examples/1 ]). @@ -161,6 +162,14 @@ values(api_v2, put) -> token => <<"my_token">>, ssl => #{enable => false} }. + +desc(influxdb_udp) -> + ?DESC("influxdb_udp"); +desc(influxdb_api_v1) -> + ?DESC("influxdb_api_v1"); +desc(influxdb_api_v2) -> + ?DESC("influxdb_api_v2"). + %% ------------------------------------------------------------------------------------------------- %% internal functions diff --git a/scripts/spellcheck b/scripts/spellcheck index 51d8d2907..f8af8c3f6 100755 --- a/scripts/spellcheck +++ b/scripts/spellcheck @@ -7,7 +7,7 @@ else SCHEMA="$1" fi -docker run -d --name langtool "ghcr.io/emqx/emqx-schema-validate:0.3.3" +docker run -d --name langtool "ghcr.io/emqx/emqx-schema-validate:0.3.5" docker exec -i langtool ./emqx_schema_validate - < "${SCHEMA}" success="$?" From 9d17bf31f812495f261b696e1d041533804f1ad7 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 9 Aug 2022 14:28:54 +0800 Subject: [PATCH 27/71] fix: InfluxDB api v1 not support uint type --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 490abfc09..83c5a4127 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -49,6 +49,12 @@ conn_bridge_examples(Method) -> values(Protocol, get) -> maps:merge(values(Protocol, post), ?METRICS_EXAMPLE); values(Protocol, post) -> + case Protocol of + "influxdb_api_v2" -> + SupportUint = <<"uint_value=${payload.uint_key}u">>; + _ -> + SupportUint = <<>> + end, #{ type => list_to_atom(Protocol), name => <<"demo">>, @@ -58,7 +64,7 @@ values(Protocol, post) -> local_topic => <<"local/topic/#">>, write_syntax => <<"${topic},clientid=${clientid}", " ", "payload=${payload},", - "${clientid}_int_value=${payload.int_key}i,", "uint_value=${payload.uint_key}u,", + "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary, "bool=${payload.bool}">> }; values(Protocol, put) -> From 12904d797fe0677fc50b487c3b5387a63aa98fb3 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 25 Jul 2022 23:51:23 +0800 Subject: [PATCH 28/71] feat(resource): first commit for batching/async/caching mechanism --- apps/emqx_resource/include/emqx_resource.hrl | 1 + .../include/emqx_resource_utils.hrl | 5 +- apps/emqx_resource/src/emqx_resource.erl | 58 +- .../src/emqx_resource_manager.erl | 30 +- apps/emqx_resource/src/emqx_resource_sup.erl | 4 +- .../emqx_resource/src/emqx_resource_utils.erl | 17 + .../src/emqx_resource_worker.erl | 252 ++++++ apps/emqx_resource/src/replayq.erl | 779 ++++++++++++++++++ 8 files changed, 1091 insertions(+), 55 deletions(-) create mode 100644 apps/emqx_resource/src/emqx_resource_utils.erl create mode 100644 apps/emqx_resource/src/emqx_resource_worker.erl create mode 100644 apps/emqx_resource/src/replayq.erl diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index dd384af7c..d6f959510 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -54,3 +54,4 @@ -type after_query_fun() :: {fun((...) -> ok), Args :: [term()]}. -define(TEST_ID_PREFIX, "_test_:"). +-define(RES_METRICS, resource_metrics). diff --git a/apps/emqx_resource/include/emqx_resource_utils.hrl b/apps/emqx_resource/include/emqx_resource_utils.hrl index 8d94746eb..3df64b1e5 100644 --- a/apps/emqx_resource/include/emqx_resource_utils.hrl +++ b/apps/emqx_resource/include/emqx_resource_utils.hrl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -define(SAFE_CALL(_EXP_), - ?SAFE_CALL(_EXP_, ok) + ?SAFE_CALL(_EXP_, {error, {_EXCLASS_, _EXCPTION_, _ST_}}) ). -define(SAFE_CALL(_EXP_, _EXP_ON_FAIL_), @@ -24,8 +24,7 @@ (_EXP_) catch _EXCLASS_:_EXCPTION_:_ST_ -> - _EXP_ON_FAIL_, - {error, {_EXCLASS_, _EXCPTION_, _ST_}} + _EXP_ON_FAIL_ end end() ). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 33f0d0a3d..793b9f446 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -83,8 +83,7 @@ stop/1, %% query the instance query/2, - %% query the instance with after_query() - query/3 + query_async/3 ]). %% Direct calls to the callback module @@ -111,6 +110,8 @@ list_group_instances/1 ]). +-export([inc_metrics_funcs/1, inc_success/1, inc_failed/1]). + -optional_callbacks([ on_query/4, on_get_status/2 @@ -150,16 +151,16 @@ is_resource_mod(Module) -> -spec query_success(after_query()) -> ok. query_success(undefined) -> ok; -query_success({OnSucc, _}) -> apply_query_after_calls(OnSucc). +query_success({OnSucc, _}) -> exec_query_after_calls(OnSucc). -spec query_failed(after_query()) -> ok. query_failed(undefined) -> ok; -query_failed({_, OnFailed}) -> apply_query_after_calls(OnFailed). +query_failed({_, OnFailed}) -> exec_query_after_calls(OnFailed). -apply_query_after_calls(Funcs) -> +exec_query_after_calls(Funcs) -> lists:foreach( - fun({Fun, Args}) -> - safe_apply(Fun, Args) + fun({Fun, Arg}) -> + emqx_resource_utils:safe_exec(Fun, Arg) end, Funcs ). @@ -243,29 +244,12 @@ reset_metrics(ResId) -> %% ================================================================================= -spec query(resource_id(), Request :: term()) -> Result :: term(). query(ResId, Request) -> - query(ResId, Request, inc_metrics_funcs(ResId)). + emqx_resource_worker:query(ResId, Request). -%% same to above, also defines what to do when the Module:on_query success or failed -%% it is the duty of the Module to apply the `after_query()` functions. --spec query(resource_id(), Request :: term(), after_query()) -> Result :: term(). -query(ResId, Request, AfterQuery) -> - case emqx_resource_manager:ets_lookup(ResId) of - {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> - %% the resource state is readonly to Module:on_query/4 - %% and the `after_query()` functions should be thread safe - ok = emqx_metrics_worker:inc(resource_metrics, ResId, matched), - try - Mod:on_query(ResId, Request, AfterQuery, ResourceState) - catch - Err:Reason:ST -> - emqx_metrics_worker:inc(resource_metrics, ResId, exception), - erlang:raise(Err, Reason, ST) - end; - {ok, _Group, _Data} -> - query_error(not_connected, <<"resource not connected">>); - {error, not_found} -> - query_error(not_found, <<"resource not found">>) - end. +-spec query_async(resource_id(), Request :: term(), emqx_resource_worker:reply_fun()) -> + ok. +query_async(ResId, Request, ReplyFun) -> + emqx_resource_worker:query_async(ResId, Request, ReplyFun). -spec start(resource_id()) -> ok | {error, Reason :: term()}. start(ResId) -> @@ -429,16 +413,16 @@ check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> %% ================================================================================= +inc_success(ResId) -> + emqx_metrics_worker:inc(?RES_METRICS, ResId, success). + +inc_failed(ResId) -> + emqx_metrics_worker:inc(?RES_METRICS, ResId, failed). + filter_instances(Filter) -> [Id || #{id := Id, mod := Mod} <- list_instances_verbose(), Filter(Id, Mod)]. inc_metrics_funcs(ResId) -> - OnFailed = [{fun emqx_metrics_worker:inc/3, [resource_metrics, ResId, failed]}], - OnSucc = [{fun emqx_metrics_worker:inc/3, [resource_metrics, ResId, success]}], + OnSucc = [{fun ?MODULE:inc_success/1, ResId}], + OnFailed = [{fun ?MODULE:inc_failed/1, ResId}], {OnSucc, OnFailed}. - -safe_apply(Func, Args) -> - ?SAFE_CALL(erlang:apply(Func, Args)). - -query_error(Reason, Msg) -> - {error, {?MODULE, #{reason => Reason, msg => Msg}}}. diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 82be9c58f..f1fa36173 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -109,9 +109,9 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) -> % The state machine will make the actual call to the callback/resource module after init ok = emqx_resource_manager_sup:ensure_child(MgrId, ResId, Group, ResourceType, Config, Opts), ok = emqx_metrics_worker:create_metrics( - resource_metrics, + ?RES_METRICS, ResId, - [matched, success, failed, exception], + [matched, success, failed, exception, resource_error], [matched] ), case maps:get(start_after_created, Opts, true) of @@ -207,12 +207,12 @@ ets_lookup(ResId) -> %% @doc Get the metrics for the specified resource get_metrics(ResId) -> - emqx_metrics_worker:get_metrics(resource_metrics, ResId). + emqx_metrics_worker:get_metrics(?RES_METRICS, ResId). %% @doc Reset the metrics for the specified resource -spec reset_metrics(resource_id()) -> ok. reset_metrics(ResId) -> - emqx_metrics_worker:reset_metrics(resource_metrics, ResId). + emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId). %% @doc Returns the data for all resources -spec list_all() -> [resource_data()] | []. @@ -298,8 +298,7 @@ handle_event({call, From}, stop, stopped, _Data) -> {keep_state_and_data, [{reply, From, ok}]}; handle_event({call, From}, stop, _State, Data) -> Result = stop_resource(Data), - UpdatedData = Data#data{status = disconnected}, - {next_state, stopped, UpdatedData, [{reply, From, Result}]}; + {next_state, stopped, Data, [{reply, From, Result}]}; % Called when a resource is to be stopped and removed. handle_event({call, From}, {remove, ClearMetrics}, _State, Data) -> handle_remove_event(From, ClearMetrics, Data); @@ -315,9 +314,10 @@ handle_event({call, From}, health_check, _State, Data) -> handle_manually_health_check(From, Data); % State: CONNECTING handle_event(enter, _OldState, connecting, Data) -> + UpdatedData = Data#data{status = connected}, insert_cache(Data#data.id, Data#data.group, Data), Actions = [{state_timeout, 0, health_check}], - {keep_state_and_data, Actions}; + {keep_state, UpdatedData, Actions}; handle_event(internal, start_resource, connecting, Data) -> start_resource(Data, undefined); handle_event(state_timeout, health_check, connecting, Data) -> @@ -326,22 +326,24 @@ handle_event(state_timeout, health_check, connecting, Data) -> %% The connected state is entered after a successful on_start/2 of the callback mod %% and successful health_checks handle_event(enter, _OldState, connected, Data) -> - insert_cache(Data#data.id, Data#data.group, Data), + UpdatedData = Data#data{status = connected}, + insert_cache(Data#data.id, Data#data.group, UpdatedData), _ = emqx_alarm:deactivate(Data#data.id), Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}], - {next_state, connected, Data, Actions}; + {next_state, connected, UpdatedData, Actions}; handle_event(state_timeout, health_check, connected, Data) -> handle_connected_health_check(Data); %% State: DISCONNECTED handle_event(enter, _OldState, disconnected, Data) -> - insert_cache(Data#data.id, Data#data.group, Data), - handle_disconnected_state_enter(Data); + UpdatedData = Data#data{status = disconnected}, + insert_cache(Data#data.id, Data#data.group, UpdatedData), + handle_disconnected_state_enter(UpdatedData); handle_event(state_timeout, auto_retry, disconnected, Data) -> start_resource(Data, undefined); %% State: STOPPED %% The stopped state is entered after the resource has been explicitly stopped handle_event(enter, _OldState, stopped, Data) -> - UpdatedData = Data#data{status = disconnected}, + UpdatedData = Data#data{status = stopped}, insert_cache(Data#data.id, Data#data.group, UpdatedData), {next_state, stopped, UpdatedData}; % Ignore all other events @@ -415,7 +417,7 @@ handle_disconnected_state_enter(Data) -> handle_remove_event(From, ClearMetrics, Data) -> stop_resource(Data), case ClearMetrics of - true -> ok = emqx_metrics_worker:clear_metrics(resource_metrics, Data#data.id); + true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok end, {stop_and_reply, normal, [{reply, From, ok}]}. @@ -433,7 +435,7 @@ start_resource(Data, From) -> _ = maybe_alarm(disconnected, Data#data.id), %% Keep track of the error reason why the connection did not work %% so that the Reason can be returned when the verification call is made. - UpdatedData = Data#data{status = disconnected, error = Reason}, + UpdatedData = Data#data{error = Reason}, Actions = maybe_reply([], From, Err), {next_state, disconnected, UpdatedData, Actions} end. diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index 1120723c3..84458f0d5 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -15,6 +15,8 @@ %%-------------------------------------------------------------------- -module(emqx_resource_sup). +-include("emqx_resource.hrl"). + -behaviour(supervisor). -export([start_link/0]). @@ -29,7 +31,7 @@ start_link() -> init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, - Metrics = emqx_metrics_worker:child_spec(resource_metrics), + Metrics = emqx_metrics_worker:child_spec(?RES_METRICS), ResourceManager = #{ diff --git a/apps/emqx_resource/src/emqx_resource_utils.erl b/apps/emqx_resource/src/emqx_resource_utils.erl new file mode 100644 index 000000000..715691d2a --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_utils.erl @@ -0,0 +1,17 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_utils). diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl new file mode 100644 index 000000000..6c3b05830 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -0,0 +1,252 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% An FIFO queue using ETS-ReplayQ as backend. + +-module(emqx_resource_worker). + +-include("emqx_resource.hrl"). +-include("emqx_resource_utils.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-behaviour(gen_statem). + +-export([ + start_link/2, + query/2, + query_async/3, + query_mfa/3 +]). + +-export([ + callback_mode/0, + init/1 +]). + +-export([do/3]). + +%% count +-define(DEFAULT_BATCH_SIZE, 100). +%% milliseconds +-define(DEFAULT_BATCH_TIME, 10). + +-define(QUERY(FROM, REQUEST), {FROM, REQUEST}). +-define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}). +-define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]). + +-type id() :: binary(). +-type request() :: term(). +-type result() :: term(). +-type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()}. +-type from() :: pid() | reply_fun(). + +-callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) -> + {{from(), result()}, NewCbState :: term()}. + +callback_mode() -> [state_functions]. + +start_link(Id, Opts) -> + gen_statem:start_link({local, name(Id)}, ?MODULE, {Id, Opts}, []). + +-spec query(id(), request()) -> ok. +query(Id, Request) -> + gen_statem:call(name(Id), {query, Request}). + +-spec query_async(id(), request(), reply_fun()) -> ok. +query_async(Id, Request, ReplyFun) -> + gen_statem:cast(name(Id), {query, Request, ReplyFun}). + +-spec name(id()) -> atom(). +name(Id) -> + Mod = atom_to_binary(?MODULE, utf8), + <>. + +disk_cache_dir(Id) -> + filename:join([emqx:data_dir(), Id, cache]). + +init({Id, Opts}) -> + BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), + Queue = + case maps:get(cache_enabled, Opts, true) of + true -> replayq:open(#{dir => disk_cache_dir(Id), seg_bytes => 10000000}); + false -> undefined + end, + St = #{ + id => Id, + batch_enabled => maps:get(batch_enabled, Opts, true), + batch_size => BatchSize, + batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), + cache_queue => Queue, + acc => [], + acc_left => BatchSize, + tref => undefined + }, + {ok, do, St}. + +do(cast, {query, Request, ReplyFun}, #{batch_enabled := true} = State) -> + do_acc(ReplyFun, Request, State); +do(cast, {query, Request, ReplyFun}, #{batch_enabled := false} = State) -> + do_query(ReplyFun, Request, State); +do({call, From}, {query, Request}, #{batch_enabled := true} = State) -> + do_acc(From, Request, State); +do({call, From}, {query, Request}, #{batch_enabled := false} = State) -> + do_query(From, Request, State); +do(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> + {keep_state, flush(St#{tref := undefined})}; +do(info, {flush, _Ref}, _St) -> + keep_state_and_data; +do(info, Info, _St) -> + ?SLOG(error, #{msg => unexpected_msg, info => Info}), + keep_state_and_data. + +do_acc(From, Request, #{acc := Acc, acc_left := Left} = St0) -> + Acc1 = [?QUERY(From, Request) | Acc], + St = St0#{acc := Acc1, acc_left := Left - 1}, + case Left =< 1 of + true -> {keep_state, flush(St)}; + false -> {keep_state, ensure_flush_timer(St)} + end. + +do_query(From, Request, #{id := Id, cache_queue := Q0} = St0) -> + Result = call_query(Id, Request), + Q1 = reply_caller(Id, Q0, ?REPLY(From, Request, Result)), + {keep_state, St0#{cache_queue := Q1}}. + +flush(#{acc := []} = St) -> + St; +flush( + #{ + id := Id, + acc := Batch, + batch_size := Size, + cache_queue := Q0 + } = St +) -> + BatchResults = call_batch_query(Id, Batch), + Q1 = batch_reply_caller(Id, Q0, BatchResults), + cancel_flush_timer( + St#{ + acc_left := Size, + acc := [], + cache_queue := Q1 + } + ). + +maybe_append_cache(undefined, _Request) -> undefined; +maybe_append_cache(Q, Request) -> replayq:append(Q, Request). + +batch_reply_caller(Id, Q, BatchResults) -> + lists:foldl( + fun(Reply, Q1) -> + reply_caller(Id, Q1, Reply) + end, + Q, + BatchResults + ). + +reply_caller(Id, Q, ?REPLY({ReplyFun, Args}, Request, Result)) when is_function(ReplyFun) -> + ?SAFE_CALL(ReplyFun(Result, Args)), + handle_query_result(Id, Q, Request, Result); +reply_caller(Id, Q, ?REPLY(From, Request, Result)) -> + gen_statem:reply(From, Result), + handle_query_result(Id, Q, Request, Result). + +handle_query_result(Id, Q, _Request, ok) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, success), + Q; +handle_query_result(Id, Q, _Request, {ok, _}) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, success), + Q; +handle_query_result(Id, Q, _Request, {error, _}) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, failed), + Q; +handle_query_result(Id, Q, Request, {error, {resource_error, #{reason := not_connected}}}) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error), + maybe_append_cache(Q, Request); +handle_query_result(Id, Q, _Request, {error, {resource_error, #{}}}) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error), + Q; +handle_query_result(Id, Q, Request, {error, {exception, _}}) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, exception), + maybe_append_cache(Q, Request). + +call_query(Id, Request) -> + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched), + case emqx_resource_manager:ets_lookup(Id) of + {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> + try Mod:on_query(Id, Request, ResourceState) of + Result -> Result + catch + Err:Reason:ST -> + ModB = atom_to_binary(Mod, utf8), + Msg = <<"call failed, func: ", ModB/binary, ":on_query/3">>, + exception_error(Reason, Msg, {Err, Reason, ST}) + end; + {ok, _Group, #{status := stopped}} -> + resource_error(stopped, <<"resource stopped or disabled">>); + {ok, _Group, _Data} -> + resource_error(not_connected, <<"resource not connected">>); + {error, not_found} -> + resource_error(not_found, <<"resource not found">>) + end. + +call_batch_query(Id, Batch) -> + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)), + case emqx_resource_manager:ets_lookup(Id) of + {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> + try Mod:on_batch_query(Id, Batch, ResourceState) of + BatchResults -> BatchResults + catch + Err:Reason:ST -> + ModB = atom_to_binary(Mod, utf8), + Msg = <<"call failed, func: ", ModB/binary, ":on_batch_query/3">>, + ?EXPAND(exception_error(Reason, Msg, {Err, Reason, ST}), Batch) + end; + {ok, _Group, _Data} -> + ?EXPAND(resource_error(not_connected, <<"resource not connected">>), Batch); + {error, not_found} -> + ?EXPAND(resource_error(not_found, <<"resource not found">>), Batch) + end. + +resource_error(Reason, Msg) -> + {error, {resource_error, #{reason => Reason, msg => Msg}}}. +exception_error(Reason, Msg, Details) -> + {error, {exception, #{reason => Reason, msg => Msg, details => Details}}}. + +%% ========================================== +ensure_flush_timer(St = #{tref := undefined, batch_time := T}) -> + Ref = make_ref(), + TRef = erlang:send_after(T, self(), {flush, Ref}), + St#{tref => {TRef, Ref}}; +ensure_flush_timer(St) -> + St. + +cancel_flush_timer(St = #{tref := undefined}) -> + St; +cancel_flush_timer(St = #{tref := {TRef, _Ref}}) -> + _ = erlang:cancel_timer(TRef), + St#{tref => undefined}. + +query_mfa(InsertMode, Request, SyncTimeout) -> + {?MODULE, query_fun(InsertMode), query_args(InsertMode, Request, SyncTimeout)}. + +query_fun(<<"sync">>) -> query; +query_fun(<<"async">>) -> query_async. + +query_args(<<"sync">>, Request, SyncTimeout) -> + [Request, SyncTimeout]; +query_args(<<"async">>, Request, _) -> + [Request]. diff --git a/apps/emqx_resource/src/replayq.erl b/apps/emqx_resource/src/replayq.erl new file mode 100644 index 000000000..437702a5e --- /dev/null +++ b/apps/emqx_resource/src/replayq.erl @@ -0,0 +1,779 @@ +-module(replayq). + +-export([open/1, close/1]). +-export([append/2, pop/2, ack/2, ack_sync/2, peek/1, overflow/1]). +-export([count/1, bytes/1, is_empty/1, is_mem_only/1]). +%% exported for troubleshooting +-export([do_read_items/2]). + +%% internal exports for beam reload +-export([committer_loop/2, default_sizer/1, default_marshaller/1]). + +-export_type([config/0, q/0, ack_ref/0, sizer/0, marshaller/0]). + +-define(NOTHING_TO_ACK, nothing_to_ack). +-define(PENDING_ACKS(Ref), {replayq_pending_acks, Ref}). + +-type segno() :: pos_integer(). +-type item() :: term(). +-type count() :: non_neg_integer(). +-type id() :: count(). +-type bytes() :: non_neg_integer(). +-type filename() :: file:filename_all(). +-type dir() :: filename(). +-type ack_ref() :: ?NOTHING_TO_ACK | {segno(), ID :: pos_integer()}. +-type sizer() :: fun((item()) -> bytes()). +-type marshaller() :: fun((item()) -> binary()). + +-type config() :: #{ + dir => dir(), + seg_bytes => bytes(), + mem_only => boolean(), + max_total_bytes => bytes(), + offload => boolean(), + sizer => sizer(), + marshaller => marshaller() +}. +%% writer cursor +-define(NO_FD, no_fd). +-type w_cur() :: #{ + segno := segno(), + bytes := bytes(), + count := count(), + fd := ?NO_FD | file:fd() +}. + +-type stats() :: #{ + bytes := bytes(), + count := count() +}. + +-opaque q() :: #{ + config := mem_only | config(), + stats := stats(), + in_mem := queue:queue(in_mem_item()), + w_cur => w_cur(), + committer => pid(), + head_segno => segno(), + sizer := sizer(), + marshaller => marshaller(), + max_total_bytes := bytes() +}. + +-define(LAYOUT_VSN_0, 0). +-define(LAYOUT_VSN_1, 1). +-define(MAGIC, 841265288). +-define(SUFFIX, "replaylog"). +-define(DEFAULT_POP_BYTES_LIMIT, 2000000). +-define(DEFAULT_POP_COUNT_LIMIT, 1000). +-define(DEFAULT_REPLAYQ_LIMIT, 2000000000). +-define(COMMIT(SEGNO, ID, From), {commit, SEGNO, ID, From}). +-define(NO_COMMIT_HIST, no_commit_hist). +-define(FIRST_SEGNO, 1). +-define(NEXT_SEGNO(N), (N + 1)). +-define(STOP, stop). +-define(MEM_ONLY_ITEM(Bytes, Item), {Bytes, Item}). +-define(DISK_CP_ITEM(Id, Bytes, Item), {Id, Bytes, Item}). + +-type in_mem_item() :: + ?MEM_ONLY_ITEM(bytes(), item()) + | ?DISK_CP_ITEM(id(), bytes(), item()). + +-spec open(config()) -> q(). +open(#{mem_only := true} = C) -> + #{ + stats => #{bytes => 0, count => 0}, + in_mem => queue:new(), + sizer => get_sizer(C), + config => mem_only, + max_total_bytes => maps:get(max_total_bytes, C, ?DEFAULT_REPLAYQ_LIMIT) + }; +open(#{dir := Dir, seg_bytes := _} = Config) -> + ok = filelib:ensure_dir(filename:join(Dir, "foo")), + Sizer = get_sizer(Config), + Marshaller = get_marshaller(Config), + IsOffload = is_offload_mode(Config), + Q = + case delete_consumed_and_list_rest(Dir) of + [] -> + %% no old segments, start over from zero + #{ + stats => #{bytes => 0, count => 0}, + w_cur => init_writer(Dir, empty, IsOffload), + committer => spawn_committer(?FIRST_SEGNO, Dir), + head_segno => ?FIRST_SEGNO, + in_mem => queue:new() + }; + Segs -> + LastSegno = lists:last(Segs), + CommitHist = get_commit_hist(Dir), + Reader = fun(Seg, Ch) -> read_items(Dir, Seg, Ch, Sizer, Marshaller) end, + HeadItems = Reader(hd(Segs), CommitHist), + #{ + stats => collect_stats(HeadItems, tl(Segs), Reader), + w_cur => init_writer(Dir, LastSegno, IsOffload), + committer => spawn_committer(hd(Segs), Dir), + head_segno => hd(Segs), + in_mem => queue:from_list(HeadItems) + } + end, + Q#{ + sizer => Sizer, + marshaller => Marshaller, + config => maps:without([sizer, marshaller], Config), + max_total_bytes => maps:get(max_total_bytes, Config, ?DEFAULT_REPLAYQ_LIMIT) + }. + +-spec close(q() | w_cur()) -> ok | {error, any()}. +close(#{config := mem_only}) -> + ok; +close(#{w_cur := W_Cur, committer := Pid} = Q) -> + MRef = erlang:monitor(process, Pid), + Pid ! ?STOP, + unlink(Pid), + receive + {'DOWN', MRef, process, Pid, _Reason} -> + ok + end, + ok = maybe_dump_back_to_disk(Q), + do_close(W_Cur). + +do_close(#{fd := ?NO_FD}) -> ok; +do_close(#{fd := Fd}) -> file:close(Fd). + +%% In case of offload mode, dump the unacked (and un-popped) on disk +%% before close. this serves as a best-effort data loss protection +maybe_dump_back_to_disk(#{config := Config} = Q) -> + case is_offload_mode(Config) of + true -> dump_back_to_disk(Q); + false -> ok + end. + +dump_back_to_disk(#{ + config := #{dir := Dir}, + head_segno := ReaderSegno, + in_mem := InMem, + marshaller := Marshaller +}) -> + IoData0 = get_unacked(process_info(self(), dictionary), ReaderSegno, Marshaller), + Items1 = queue:to_list(InMem), + IoData1 = lists:map(fun(?DISK_CP_ITEM(_, _, I)) -> make_iodata(I, Marshaller) end, Items1), + %% ensure old segment file is deleted + ok = ensure_deleted(filename(Dir, ReaderSegno)), + %% rewrite the segment with what's currently in memory + IoData = [IoData0, IoData1], + case iolist_size(IoData) > 0 of + true -> + #{fd := Fd} = open_segment(Dir, ReaderSegno), + ok = file:write(Fd, [IoData0, IoData1]), + ok = file:close(Fd); + false -> + %% nothing to write + ok + end. + +get_unacked({dictionary, Dict}, ReaderSegno, Marshaller) -> + F = fun + ({?PENDING_ACKS(AckRef), Items}) -> + erase(?PENDING_ACKS(AckRef)), + {Segno, Id} = AckRef, + Segno =:= ReaderSegno andalso + {true, {Id, Items}}; + (_) -> + false + end, + Pendings0 = lists:filtermap(F, Dict), + Pendings = lists:keysort(1, Pendings0), + do_get_unacked(Pendings, Marshaller). + +do_get_unacked([], _Marshaller) -> + []; +do_get_unacked([{_, Items} | Rest], Marshaller) -> + [ + [make_iodata(I, Marshaller) || I <- Items] + | do_get_unacked(Rest, Marshaller) + ]. + +-spec append(q(), [item()]) -> q(). +append(Q, []) -> + Q; +append( + #{ + config := mem_only, + in_mem := InMem, + stats := #{bytes := Bytes0, count := Count0}, + sizer := Sizer + } = Q, + Items0 +) -> + {CountDiff, BytesDiff, Items} = transform(false, Items0, Sizer), + + Stats = #{count => Count0 + CountDiff, bytes => Bytes0 + BytesDiff}, + Q#{ + stats := Stats, + in_mem := append_in_mem(Items, InMem) + }; +append( + #{ + config := #{seg_bytes := BytesLimit, dir := Dir} = Config, + stats := #{bytes := Bytes0, count := Count0}, + w_cur := #{count := CountInSeg, segno := WriterSegno} = W_Cur0, + head_segno := ReaderSegno, + sizer := Sizer, + marshaller := Marshaller, + in_mem := HeadItems0 + } = Q, + Items0 +) -> + IoData = lists:map(fun(I) -> make_iodata(I, Marshaller) end, Items0), + {CountDiff, BytesDiff, Items} = transform(CountInSeg + 1, Items0, Sizer), + TotalBytes = Bytes0 + BytesDiff, + Stats = #{count => Count0 + CountDiff, bytes => TotalBytes}, + IsOffload = is_offload_mode(Config), + W_Cur1 = do_append(W_Cur0, CountDiff, BytesDiff, IoData), + W_Cur = + case is_segment_full(W_Cur1, TotalBytes, BytesLimit, ReaderSegno, IsOffload) of + true -> + ok = do_close(W_Cur1), + %% get ready for the next append + open_segment(Dir, ?NEXT_SEGNO(WriterSegno)); + false -> + W_Cur1 + end, + HeadItems = + case ReaderSegno =:= WriterSegno of + true -> append_in_mem(Items, HeadItems0); + false -> HeadItems0 + end, + Q#{ + stats := Stats, + w_cur := W_Cur, + in_mem := HeadItems + }. + +%% @doc pop out at least one item from the queue. +%% volume limited by `bytes_limit' and `count_limit'. +-spec pop(q(), #{bytes_limit => bytes(), count_limit => count()}) -> + {q(), ack_ref(), [item()]}. +pop(Q, Opts) -> + Bytes = maps:get(bytes_limit, Opts, ?DEFAULT_POP_BYTES_LIMIT), + Count = maps:get(count_limit, Opts, ?DEFAULT_POP_COUNT_LIMIT), + true = (Count > 0), + pop(Q, Bytes, Count, ?NOTHING_TO_ACK, []). + +%% @doc peek the queue front item. +-spec peek(q()) -> empty | item(). +peek(#{in_mem := HeadItems}) -> + case queue:peek(HeadItems) of + empty -> empty; + {value, ?MEM_ONLY_ITEM(_, Item)} -> Item; + {value, ?DISK_CP_ITEM(_, _, Item)} -> Item + end. + +%% @doc Asynch-ly write the consumed item Segment number + ID to a file. +-spec ack(q(), ack_ref()) -> ok. +ack(_, ?NOTHING_TO_ACK) -> + ok; +ack(#{committer := Pid}, {Segno, Id} = AckRef) -> + _ = erlang:erase(?PENDING_ACKS(AckRef)), + Pid ! ?COMMIT(Segno, Id, false), + ok. + +%% @hidden Synced ack, for deterministic tests only +-spec ack_sync(q(), ack_ref()) -> ok. +ack_sync(_, ?NOTHING_TO_ACK) -> + ok; +ack_sync(#{committer := Pid}, {Segno, Id} = AckRef) -> + _ = erlang:erase(?PENDING_ACKS(AckRef)), + Ref = make_ref(), + Pid ! ?COMMIT(Segno, Id, {self(), Ref}), + receive + {Ref, ok} -> ok + end. + +-spec count(q()) -> count(). +count(#{stats := #{count := Count}}) -> Count. + +-spec bytes(q()) -> bytes(). +bytes(#{stats := #{bytes := Bytes}}) -> Bytes. + +is_empty(#{config := mem_only, in_mem := All}) -> + queue:is_empty(All); +is_empty( + #{ + w_cur := #{segno := WriterSegno}, + head_segno := ReaderSegno, + in_mem := HeadItems + } = Q +) -> + Result = ((WriterSegno =:= ReaderSegno) andalso queue:is_empty(HeadItems)), + %% assert + Result = (count(Q) =:= 0). + +%% @doc Returns number of bytes the size of the queue has exceeded +%% total bytes limit. Result is negative when it is not overflow. +-spec overflow(q()) -> integer(). +overflow(#{ + max_total_bytes := MaxTotalBytes, + stats := #{bytes := Bytes} +}) -> + Bytes - MaxTotalBytes. + +-spec is_mem_only(q()) -> boolean(). +is_mem_only(#{config := mem_only}) -> + true; +is_mem_only(_) -> + false. + +%% internals ========================================================= + +transform(Id, Items, Sizer) -> + transform(Id, Items, Sizer, 0, 0, []). + +transform(_Id, [], _Sizer, Count, Bytes, Acc) -> + {Count, Bytes, lists:reverse(Acc)}; +transform(Id, [Item0 | Rest], Sizer, Count, Bytes, Acc) -> + Size = Sizer(Item0), + {NextId, Item} = + case Id of + false -> {false, ?MEM_ONLY_ITEM(Size, Item0)}; + N -> {N + 1, ?DISK_CP_ITEM(Id, Size, Item0)} + end, + transform(NextId, Rest, Sizer, Count + 1, Bytes + Size, [Item | Acc]). + +append_in_mem([], Q) -> Q; +append_in_mem([Item | Rest], Q) -> append_in_mem(Rest, queue:in(Item, Q)). + +pop(Q, _Bytes, 0, AckRef, Acc) -> + Result = lists:reverse(Acc), + ok = maybe_save_pending_acks(AckRef, Q, Result), + {Q, AckRef, Result}; +pop(#{config := Cfg} = Q, Bytes, Count, AckRef, Acc) -> + case is_empty(Q) of + true -> + {Q, AckRef, lists:reverse(Acc)}; + false when Cfg =:= mem_only -> + pop_mem(Q, Bytes, Count, Acc); + false -> + pop2(Q, Bytes, Count, AckRef, Acc) + end. + +pop_mem( + #{ + in_mem := InMem, + stats := #{count := TotalCount, bytes := TotalBytes} = Stats + } = Q, + Bytes, + Count, + Acc +) -> + case queue:out(InMem) of + {{value, ?MEM_ONLY_ITEM(Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] -> + {Q, ?NOTHING_TO_ACK, lists:reverse(Acc)}; + {{value, ?MEM_ONLY_ITEM(Sz, Item)}, Rest} -> + NewQ = Q#{ + in_mem := Rest, + stats := Stats#{ + count := TotalCount - 1, + bytes := TotalBytes - Sz + } + }, + pop(NewQ, Bytes - Sz, Count - 1, ?NOTHING_TO_ACK, [Item | Acc]) + end. + +pop2( + #{ + head_segno := ReaderSegno, + in_mem := HeadItems, + stats := #{count := TotalCount, bytes := TotalBytes} = Stats, + w_cur := #{segno := WriterSegno} + } = Q, + Bytes, + Count, + AckRef, + Acc +) -> + case queue:out(HeadItems) of + {{value, ?DISK_CP_ITEM(_, Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] -> + %% taking the head item would cause exceeding size limit + {Q, AckRef, lists:reverse(Acc)}; + {{value, ?DISK_CP_ITEM(Id, Sz, Item)}, Rest} -> + Q1 = Q#{ + in_mem := Rest, + stats := Stats#{ + count := TotalCount - 1, + bytes := TotalBytes - Sz + } + }, + %% read the next segment in case current is drained + NewQ = + case queue:is_empty(Rest) andalso ReaderSegno < WriterSegno of + true -> read_next_seg(Q1); + false -> Q1 + end, + NewAckRef = {ReaderSegno, Id}, + pop(NewQ, Bytes - Sz, Count - 1, NewAckRef, [Item | Acc]) + end. + +%% due to backward compatibility reasons for the ack api +%% we ca nnot track pending acks in q() -- reason to use process dictionary +maybe_save_pending_acks(?NOTHING_TO_ACK, _, _) -> + ok; +maybe_save_pending_acks(AckRef, #{config := Config}, Items) -> + case is_offload_mode(Config) of + true -> + _ = erlang:put(?PENDING_ACKS(AckRef), Items), + ok; + false -> + ok + end. + +read_next_seg( + #{ + config := #{dir := Dir} = Config, + head_segno := ReaderSegno, + w_cur := #{segno := WriterSegno, fd := Fd} = WCur0, + sizer := Sizer, + marshaller := Marshaller + } = Q +) -> + NextSegno = ReaderSegno + 1, + %% reader has caught up to latest segment + case NextSegno =:= WriterSegno of + true -> + %% force flush to disk so the next read can get all bytes + ok = file:sync(Fd); + false -> + ok + end, + IsOffload = is_offload_mode(Config), + WCur = + case IsOffload andalso NextSegno =:= WriterSegno of + true -> + %% reader has caught up to latest segment in offload mode, + %% close the writer's fd. Continue in mem-only mode for the head segment + ok = do_close(WCur0), + WCur0#{fd := ?NO_FD}; + false -> + WCur0 + end, + NextSegItems = read_items(Dir, NextSegno, ?NO_COMMIT_HIST, Sizer, Marshaller), + Q#{ + head_segno := NextSegno, + in_mem := queue:from_list(NextSegItems), + w_cur := WCur + }. + +delete_consumed_and_list_rest(Dir0) -> + Dir = unicode:characters_to_list(Dir0), + Segnos0 = lists:sort([parse_segno(N) || N <- filelib:wildcard("*." ?SUFFIX, Dir)]), + {SegnosToDelete, Segnos} = find_segnos_to_delete(Dir, Segnos0), + ok = lists:foreach(fun(Segno) -> ensure_deleted(filename(Dir, Segno)) end, SegnosToDelete), + case Segnos of + [] -> + %% delete commit file in case there is no segments left + %% segment number will start from 0 again. + ensure_deleted(commit_filename(Dir)), + []; + X -> + X + end. + +find_segnos_to_delete(Dir, Segnos) -> + CommitHist = get_commit_hist(Dir), + do_find_segnos_to_delete(Dir, Segnos, CommitHist). + +do_find_segnos_to_delete(_Dir, Segnos, ?NO_COMMIT_HIST) -> + {[], Segnos}; +do_find_segnos_to_delete(Dir, Segnos0, {CommittedSegno, CommittedId}) -> + {SegnosToDelete, Segnos} = lists:partition(fun(N) -> N < CommittedSegno end, Segnos0), + case + Segnos =/= [] andalso + hd(Segnos) =:= CommittedSegno andalso + is_all_consumed(Dir, CommittedSegno, CommittedId) + of + true -> + %% assert + CommittedSegno = hd(Segnos), + %% all items in the oldest segment have been consumed, + %% no need to keep this segment + {[CommittedSegno | SegnosToDelete], tl(Segnos)}; + _ -> + {SegnosToDelete, Segnos} + end. + +%% ALL items are consumed if the committed item ID is no-less than the number +%% of items in this segment +is_all_consumed(Dir, CommittedSegno, CommittedId) -> + CommittedId >= erlang:length(do_read_items(Dir, CommittedSegno)). + +ensure_deleted(Filename) -> + case file:delete(Filename) of + ok -> ok; + {error, enoent} -> ok + end. + +%% The committer writes consumer's acked segmeng number + item ID +%% to a file. The file is only read at start/restart. +spawn_committer(ReaderSegno, Dir) -> + Name = iolist_to_binary(filename:join([Dir, committer])), + %% register a name to avoid having two committers spawned for the same dir + RegName = binary_to_atom(Name, utf8), + Pid = erlang:spawn_link(fun() -> committer_loop(ReaderSegno, Dir) end), + true = erlang:register(RegName, Pid), + Pid. + +committer_loop(ReaderSegno, Dir) -> + receive + ?COMMIT(Segno0, Id0, false) -> + {Segno, Id} = collect_async_commits(Segno0, Id0), + ok = handle_commit(ReaderSegno, Dir, Segno, Id, false), + ?MODULE:committer_loop(Segno, Dir); + ?COMMIT(Segno, Id, From) -> + ok = handle_commit(ReaderSegno, Dir, Segno, Id, From), + ?MODULE:committer_loop(Segno, Dir); + ?STOP -> + ok; + Msg -> + exit({replayq_committer_unkown_msg, Msg}) + after 200 -> + ?MODULE:committer_loop(ReaderSegno, Dir) + end. + +handle_commit(ReaderSegno, Dir, Segno, Id, From) -> + IoData = io_lib:format("~p.\n", [#{segno => Segno, id => Id}]), + ok = do_commit(Dir, IoData), + case Segno > ReaderSegno of + true -> + SegnosToDelete = lists:seq(ReaderSegno, Segno - 1), + lists:foreach(fun(N) -> ok = ensure_deleted(filename(Dir, N)) end, SegnosToDelete); + false -> + ok + end, + ok = reply_ack_ok(From). + +%% Collect async acks which are already sent in the mailbox, +%% and keep only the last one for the current segment. +collect_async_commits(Segno, Id) -> + receive + ?COMMIT(Segno, AnotherId, false) -> + collect_async_commits(Segno, AnotherId) + after 0 -> + {Segno, Id} + end. + +reply_ack_ok({Pid, Ref}) -> + Pid ! {Ref, ok}, + ok; +reply_ack_ok(_) -> + ok. + +get_commit_hist(Dir) -> + CommitFile = commit_filename(Dir), + case filelib:is_regular(CommitFile) of + true -> + {ok, [#{segno := Segno, id := Id}]} = file:consult(CommitFile), + {Segno, Id}; + false -> + ?NO_COMMIT_HIST + end. + +do_commit(Dir, IoData) -> + TmpName = commit_filename(Dir, "COMMIT.tmp"), + Name = commit_filename(Dir), + ok = file:write_file(TmpName, IoData), + ok = file:rename(TmpName, Name). + +commit_filename(Dir) -> + commit_filename(Dir, "COMMIT"). + +commit_filename(Dir, Name) -> + filename:join([Dir, Name]). + +do_append( + #{fd := ?NO_FD, bytes := Bytes0, count := Count0} = Cur, + Count, + Bytes, + _IoData +) -> + %% offload mode, fd is not initialized yet + Cur#{ + bytes => Bytes0 + Bytes, + count => Count0 + Count + }; +do_append( + #{fd := Fd, bytes := Bytes0, count := Count0} = Cur, + Count, + Bytes, + IoData +) -> + ok = file:write(Fd, IoData), + Cur#{ + bytes => Bytes0 + Bytes, + count => Count0 + Count + }. + +read_items(Dir, Segno, CommitHist, Sizer, Marshaller) -> + Items0 = do_read_items(Dir, Segno), + Items = + case CommitHist of + ?NO_COMMIT_HIST -> + %% no commit hist, return all + Items0; + {CommitedSegno, _} when CommitedSegno < Segno -> + %% committed at an older segment + Items0; + {Segno, CommittedId} -> + %% committed at current segment keep only the tail + {_, R} = lists:splitwith(fun({I, _}) -> I =< CommittedId end, Items0), + R + end, + lists:map( + fun({Id, Bin}) -> + Item = Marshaller(Bin), + Size = Sizer(Item), + ?DISK_CP_ITEM(Id, Size, Item) + end, + Items + ). + +do_read_items(Dir, Segno) -> + Filename = filename(Dir, Segno), + {ok, Bin} = file:read_file(Filename), + case parse_items(Bin, 1, []) of + {Items, <<>>} -> + Items; + {Items, Corrupted} -> + error_logger:error_msg( + "corrupted replayq log: ~s, skipped ~p bytes", + [Filename, size(Corrupted)] + ), + Items + end. + +parse_items(<<>>, _Id, Acc) -> + {lists:reverse(Acc), <<>>}; +parse_items( + <> = All, + Id, + Acc +) -> + case CRC =:= erlang:crc32(Item) of + true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]); + false -> {lists:reverse(Acc), All} + end; +parse_items( + <> = All, + Id, + Acc +) -> + case CRC =:= erlang:crc32(Item) andalso Item =/= <<>> of + true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]); + false -> {lists:reverse(Acc), All} + end; +parse_items(Corrupted, _Id, Acc) -> + {lists:reverse(Acc), Corrupted}. + +make_iodata(Item0, Marshaller) -> + Item = Marshaller(Item0), + Size = size(Item), + CRC = erlang:crc32(Item), + [ + <>, + Item + ]. + +collect_stats(HeadItems, SegsOnDisk, Reader) -> + ItemF = fun(?DISK_CP_ITEM(_Id, Sz, _Item), {B, C}) -> + {B + Sz, C + 1} + end, + Acc0 = lists:foldl(ItemF, {0, 0}, HeadItems), + {Bytes, Count} = + lists:foldl( + fun(Segno, Acc) -> + Items = Reader(Segno, ?NO_COMMIT_HIST), + lists:foldl(ItemF, Acc, Items) + end, + Acc0, + SegsOnDisk + ), + #{bytes => Bytes, count => Count}. + +parse_segno(Filename) -> + [Segno, ?SUFFIX] = string:tokens(Filename, "."), + list_to_integer(Segno). + +filename(Dir, Segno) -> + Name = lists:flatten(io_lib:format("~10.10.0w." ?SUFFIX, [Segno])), + filename:join(Dir, Name). + +%% open the current segment for write if it is empty +%% otherwise rollout to the next segment +-spec init_writer(dir(), empty | segno(), boolean()) -> w_cur(). +init_writer(_Dir, empty, true) -> + %% clean start for offload mode + #{fd => ?NO_FD, segno => ?FIRST_SEGNO, bytes => 0, count => 0}; +init_writer(Dir, empty, false) -> + open_segment(Dir, ?FIRST_SEGNO); +init_writer(Dir, Segno, _IsOffload) when is_number(Segno) -> + Filename = filename(Dir, Segno), + case filelib:file_size(Filename) of + 0 -> open_segment(Dir, Segno); + _ -> open_segment(Dir, ?NEXT_SEGNO(Segno)) + end. + +-spec open_segment(dir(), segno()) -> w_cur(). +open_segment(Dir, Segno) -> + Filename = filename(Dir, Segno), + %% raw so there is no need to go through the single gen_server file_server + {ok, Fd} = file:open(Filename, [raw, read, write, binary, delayed_write]), + #{fd => Fd, segno => Segno, bytes => 0, count => 0}. + +get_sizer(C) -> + maps:get(sizer, C, fun ?MODULE:default_sizer/1). + +get_marshaller(C) -> + maps:get(marshaller, C, fun ?MODULE:default_marshaller/1). + +is_offload_mode(Config) when is_map(Config) -> + maps:get(offload, Config, false). + +default_sizer(I) when is_binary(I) -> erlang:size(I). + +default_marshaller(I) when is_binary(I) -> I. + +is_segment_full( + #{segno := WriterSegno, bytes := SegmentBytes}, + TotalBytes, + SegmentBytesLimit, + ReaderSegno, + true +) -> + %% in offload mode, when reader is lagging behind, we try + %% writer rolls to a new segment when file size limit is reached + %% when reader is reading off from the same segment as writer + %% i.e. the in memory queue, only start writing to segment file + %% when total bytes (in memory) is reached segment limit + %% + %% NOTE: we never shrink segment bytes, even when popping out + %% from the in-memory queue. + case ReaderSegno < WriterSegno of + true -> SegmentBytes >= SegmentBytesLimit; + false -> TotalBytes >= SegmentBytesLimit + end; +is_segment_full( + #{bytes := SegmentBytes}, + _TotalBytes, + SegmentBytesLimit, + _ReaderSegno, + false +) -> + %% here we check if segment size is greater than segment size limit + %% after append based on the assumption that the items are usually + %% very small in size comparing to segment size. + %% We can change implementation to split items list to avoid + %% segment overflow if really necessary + SegmentBytes >= SegmentBytesLimit. From d8d8d674e4ad7314e87926a9a488b5db82008583 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 27 Jul 2022 00:37:17 +0800 Subject: [PATCH 29/71] feat(resource): start emqx_resource_worker in pools --- .../src/emqx_resource_manager.erl | 9 +- apps/emqx_resource/src/emqx_resource_sup.erl | 80 ++++- .../src/emqx_resource_worker.erl | 316 +++++++++++------- 3 files changed, 278 insertions(+), 127 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index f1fa36173..ee18a6836 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -111,12 +111,15 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) -> ok = emqx_metrics_worker:create_metrics( ?RES_METRICS, ResId, - [matched, success, failed, exception, resource_error], + [matched, success, failed, exception, resource_down], [matched] ), case maps:get(start_after_created, Opts, true) of - true -> wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)); - false -> ok + true -> + ok = emqx_resource_sup:start_workers(ResId, Opts), + wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)); + false -> + ok end, ok. diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index 84458f0d5..14db50d01 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -19,13 +19,10 @@ -behaviour(supervisor). --export([start_link/0]). +-export([start_link/0, start_workers/2, stop_workers/2]). -export([init/1]). -%% set a very large pool size in case all the workers busy --define(POOL_SIZE, 64). - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -43,3 +40,78 @@ init([]) -> modules => [emqx_resource_manager_sup] }, {ok, {SupFlags, [Metrics, ResourceManager]}}. + +start_workers(ResId, Opts) -> + PoolSize = pool_size(Opts), + _ = ensure_worker_pool(ResId, hash, [{size, PoolSize}]), + lists:foreach( + fun(Idx) -> + _ = ensure_worker_added(ResId, {ResId, Idx}, Idx), + ok = ensure_worker_started(ResId, Idx, Opts) + end, + lists:seq(1, PoolSize) + ). + +stop_workers(ResId, Opts) -> + PoolSize = pool_size(Opts), + lists:foreach( + fun(Idx) -> + ok = ensure_worker_stopped(ResId, Idx), + ok = ensure_worker_removed(ResId, {ResId, Idx}) + end, + lists:seq(1, PoolSize) + ), + _ = gproc_pool:delete(ResId), + ok. + +pool_size(Opts) -> + maps:get(worker_pool_size, Opts, erlang:system_info(schedulers_online)). + +ensure_worker_pool(Pool, Type, Opts) -> + try + gproc_pool:new(Pool, Type, Opts) + catch + error:exists -> ok + end, + ok. + +ensure_worker_added(Pool, Name, Slot) -> + try + gproc_pool:add_worker(Pool, Name, Slot) + catch + error:exists -> ok + end, + ok. + +ensure_worker_removed(Pool, Name) -> + _ = gproc_pool:remove_worker(Pool, Name), + ok. + +-define(CHILD_ID(MOD, RESID, INDEX), {MOD, RESID, INDEX}). +ensure_worker_started(ResId, Idx, Opts) -> + Mod = emqx_resource_worker, + Spec = #{ + id => ?CHILD_ID(Mod, ResId, Idx), + start => {Mod, start_link, [ResId, Idx, Opts]}, + restart => transient, + shutdown => 5000, + type => worker, + modules => [Mod] + }, + case supervisor:start_child(emqx_resource_sup, Spec) of + {ok, _Pid} -> ok; + {error, {already_started, _}} -> ok; + {error, already_present} -> ok; + {error, _} = Err -> Err + end. + +ensure_worker_stopped(ResId, Idx) -> + ChildId = ?CHILD_ID(emqx_resource_worker, ResId, Idx), + case supervisor:terminate_child(emqx_resource_sup, ChildId) of + ok -> + supervisor:delete_child(emqx_resource_sup, ChildId); + {error, not_found} -> + ok; + {error, Reason} -> + {error, Reason} + end. diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 6c3b05830..3c5c7eefe 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -14,7 +14,8 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% An FIFO queue using ETS-ReplayQ as backend. +%% This module implements async message sending, disk message queuing, +%% and message batching using ReplayQ. -module(emqx_resource_worker). @@ -25,18 +26,23 @@ -behaviour(gen_statem). -export([ - start_link/2, - query/2, - query_async/3, - query_mfa/3 + start_link/3, + query/3, + query_async/4, + block/1, + resume/1 ]). -export([ callback_mode/0, - init/1 + init/1, + terminate/2, + code_change/3 ]). --export([do/3]). +-export([running/3, blocked/3]). + +-define(RESUME_INTERVAL, 15000). %% count -define(DEFAULT_BATCH_SIZE, 100). @@ -47,72 +53,136 @@ -define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}). -define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]). +-define(RESOURCE_ERROR(Reason, Msg), {error, {resource_error, #{reason => Reason, msg => Msg}}}). +-define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}). + -type id() :: binary(). -type request() :: term(). -type result() :: term(). --type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()}. +-type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined. -type from() :: pid() | reply_fun(). -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) -> {{from(), result()}, NewCbState :: term()}. -callback_mode() -> [state_functions]. +callback_mode() -> [state_functions, state_enter]. -start_link(Id, Opts) -> - gen_statem:start_link({local, name(Id)}, ?MODULE, {Id, Opts}, []). +start_link(Id, Index, Opts) -> + gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []). --spec query(id(), request()) -> ok. -query(Id, Request) -> - gen_statem:call(name(Id), {query, Request}). +-spec query(id(), term(), request()) -> ok. +query(Id, Key, Request) -> + gen_statem:call(pick(Id, Key), {query, Request}). --spec query_async(id(), request(), reply_fun()) -> ok. -query_async(Id, Request, ReplyFun) -> - gen_statem:cast(name(Id), {query, Request, ReplyFun}). +-spec query_async(id(), term(), request(), reply_fun()) -> ok. +query_async(Id, Key, Request, ReplyFun) -> + gen_statem:cast(pick(Id, Key), {query, Request, ReplyFun}). --spec name(id()) -> atom(). -name(Id) -> - Mod = atom_to_binary(?MODULE, utf8), - <>. +-spec block(pid() | atom()) -> ok. +block(ServerRef) -> + gen_statem:cast(ServerRef, block). -disk_cache_dir(Id) -> - filename:join([emqx:data_dir(), Id, cache]). +-spec resume(pid() | atom()) -> ok. +resume(ServerRef) -> + gen_statem:cast(ServerRef, resume). -init({Id, Opts}) -> +init({Id, Index, Opts}) -> + true = gproc_pool:connect_worker(Id, {Id, Index}), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), Queue = - case maps:get(cache_enabled, Opts, true) of - true -> replayq:open(#{dir => disk_cache_dir(Id), seg_bytes => 10000000}); + case maps:get(queue_enabled, Opts, true) of + true -> replayq:open(#{dir => disk_queue_dir(Id), seg_bytes => 10000000}); false -> undefined end, St = #{ id => Id, + index => Index, batch_enabled => maps:get(batch_enabled, Opts, true), batch_size => BatchSize, batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), - cache_queue => Queue, + queue => Queue, acc => [], acc_left => BatchSize, tref => undefined }, - {ok, do, St}. + {ok, blocked, St, {next_event, cast, resume}}. -do(cast, {query, Request, ReplyFun}, #{batch_enabled := true} = State) -> - do_acc(ReplyFun, Request, State); -do(cast, {query, Request, ReplyFun}, #{batch_enabled := false} = State) -> - do_query(ReplyFun, Request, State); -do({call, From}, {query, Request}, #{batch_enabled := true} = State) -> - do_acc(From, Request, State); -do({call, From}, {query, Request}, #{batch_enabled := false} = State) -> - do_query(From, Request, State); -do(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> - {keep_state, flush(St#{tref := undefined})}; -do(info, {flush, _Ref}, _St) -> +running(enter, _, _St) -> keep_state_and_data; -do(info, Info, _St) -> +running(cast, resume, _St) -> + keep_state_and_data; +running(cast, block, St) -> + {next_state, block, St}; +running(cast, {query, Request, ReplyFun}, St) -> + query_or_acc(ReplyFun, Request, St); +running({call, From}, {query, Request}, St) -> + query_or_acc(From, Request, St); +running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> + {keep_state, flush(St#{tref := undefined})}; +running(info, {flush, _Ref}, _St) -> + keep_state_and_data; +running(info, Info, _St) -> ?SLOG(error, #{msg => unexpected_msg, info => Info}), keep_state_and_data. -do_acc(From, Request, #{acc := Acc, acc_left := Left} = St0) -> +blocked(enter, _, _St) -> + keep_state_and_data; +blocked(cast, block, _St) -> + keep_state_and_data; +blocked(cast, resume, St) -> + do_resume(St); +blocked(state_timeout, resume, St) -> + do_resume(St); +blocked(cast, {query, Request, ReplyFun}, St) -> + handle_blocked(ReplyFun, Request, St); +blocked({call, From}, {query, Request}, St) -> + handle_blocked(From, Request, St). + +terminate(_Reason, #{id := Id, index := Index}) -> + gproc_pool:disconnect_worker(Id, {Id, Index}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%============================================================================== +pick(Id, Key) -> + Pid = gproc_pool:pick_worker(Id, Key), + case is_pid(Pid) of + true -> Pid; + false -> error({failed_to_pick_worker, {Id, Key}}) + end. + +do_resume(#{queue := undefined} = St) -> + {next_state, running, St}; +do_resume(#{queue := Q, id := Id} = St) -> + case replayq:peek(Q) of + empty -> + {next_state, running, St, {state_timeout, ?RESUME_INTERVAL, resume}}; + First -> + Result = call_query(Id, First), + case handle_query_result(Id, false, Result) of + %% Send failed because resource down + true -> + {keep_state, St}; + %% Send ok or failed but the resource is working + false -> + %% We Send 'resume' to the end of the mailbox to give the worker + %% a chance to process 'query' requests. + {keep_state, St#{queue => drop_head(Q)}, {state_timeout, 0, resume}} + end + end. + +drop_head(Q) -> + {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}), + ok = replayq:ack(Q1, AckRef), + Q1. + +query_or_acc(From, Request, #{batch_enabled := true} = St) -> + acc_query(From, Request, St); +query_or_acc(From, Request, #{batch_enabled := false} = St) -> + send_query(From, Request, St). + +acc_query(From, Request, #{acc := Acc, acc_left := Left} = St0) -> Acc1 = [?QUERY(From, Request) | Acc], St = St0#{acc := Acc1, acc_left := Left - 1}, case Left =< 1 of @@ -120,10 +190,19 @@ do_acc(From, Request, #{acc := Acc, acc_left := Left} = St0) -> false -> {keep_state, ensure_flush_timer(St)} end. -do_query(From, Request, #{id := Id, cache_queue := Q0} = St0) -> +send_query(From, Request, #{id := Id, queue := Q} = St) -> Result = call_query(Id, Request), - Q1 = reply_caller(Id, Q0, ?REPLY(From, Request, Result)), - {keep_state, St0#{cache_queue := Q1}}. + case reply_caller(Id, Q, ?REPLY(From, Request, Result)) of + true -> + {keep_state, St#{queue := maybe_append_queue(Q, [Request])}}; + false -> + {next_state, blocked, St} + end. + +handle_blocked(From, Request, #{id := Id, queue := Q} = St) -> + Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), + _ = reply_caller(Id, Q, ?REPLY(From, Request, Error)), + {keep_state, St#{queue := maybe_append_queue(Q, [Request])}}. flush(#{acc := []} = St) -> St; @@ -132,101 +211,109 @@ flush( id := Id, acc := Batch, batch_size := Size, - cache_queue := Q0 + queue := Q0 } = St ) -> - BatchResults = call_batch_query(Id, Batch), - Q1 = batch_reply_caller(Id, Q0, BatchResults), - cancel_flush_timer( - St#{ - acc_left := Size, - acc := [], - cache_queue := Q1 - } - ). + BatchResults = maybe_expand_batch_result(call_batch_query(Id, Batch), Batch), + St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}), + case batch_reply_caller(Id, BatchResults) of + true -> + Q1 = maybe_append_queue(Q0, [Request || ?QUERY(_, Request) <- Batch]), + {keep_state, St1#{queue := Q1}}; + false -> + {next_state, blocked, St1} + end. -maybe_append_cache(undefined, _Request) -> undefined; -maybe_append_cache(Q, Request) -> replayq:append(Q, Request). +maybe_append_queue(undefined, _Query) -> undefined; +maybe_append_queue(Q, Query) -> replayq:append(Q, Query). -batch_reply_caller(Id, Q, BatchResults) -> +batch_reply_caller(Id, BatchResults) -> lists:foldl( - fun(Reply, Q1) -> - reply_caller(Id, Q1, Reply) + fun(Reply, BlockWorker) -> + reply_caller(Id, BlockWorker, Reply) end, - Q, + false, BatchResults ). -reply_caller(Id, Q, ?REPLY({ReplyFun, Args}, Request, Result)) when is_function(ReplyFun) -> +reply_caller(Id, BlockWorker, ?REPLY(undefined, _, Result)) -> + handle_query_result(Id, BlockWorker, Result); +reply_caller(Id, BlockWorker, ?REPLY({ReplyFun, Args}, _, Result)) -> ?SAFE_CALL(ReplyFun(Result, Args)), - handle_query_result(Id, Q, Request, Result); -reply_caller(Id, Q, ?REPLY(From, Request, Result)) -> + handle_query_result(Id, BlockWorker, Result); +reply_caller(Id, BlockWorker, ?REPLY(From, _, Result)) -> gen_statem:reply(From, Result), - handle_query_result(Id, Q, Request, Result). + handle_query_result(Id, BlockWorker, Result). -handle_query_result(Id, Q, _Request, ok) -> +handle_query_result(Id, BlockWorker, ok) -> emqx_metrics_worker:inc(?RES_METRICS, Id, success), - Q; -handle_query_result(Id, Q, _Request, {ok, _}) -> + BlockWorker; +handle_query_result(Id, BlockWorker, {ok, _}) -> emqx_metrics_worker:inc(?RES_METRICS, Id, success), - Q; -handle_query_result(Id, Q, _Request, {error, _}) -> - emqx_metrics_worker:inc(?RES_METRICS, Id, failed), - Q; -handle_query_result(Id, Q, Request, {error, {resource_error, #{reason := not_connected}}}) -> - emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error), - maybe_append_cache(Q, Request); -handle_query_result(Id, Q, _Request, {error, {resource_error, #{}}}) -> - emqx_metrics_worker:inc(?RES_METRICS, Id, resource_error), - Q; -handle_query_result(Id, Q, Request, {error, {exception, _}}) -> + BlockWorker; +handle_query_result(Id, BlockWorker, ?RESOURCE_ERROR_M(exception, _)) -> emqx_metrics_worker:inc(?RES_METRICS, Id, exception), - maybe_append_cache(Q, Request). + BlockWorker; +handle_query_result(_Id, _, ?RESOURCE_ERROR_M(NotWorking, _)) when + NotWorking == not_connected; NotWorking == blocked +-> + true; +handle_query_result(_Id, BlockWorker, ?RESOURCE_ERROR_M(_, _)) -> + BlockWorker; +handle_query_result(Id, BlockWorker, {error, _}) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, failed), + BlockWorker; +handle_query_result(Id, _BlockWorker, {resource_down, _}) -> + emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down), + true. call_query(Id, Request) -> - ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched), + do_call_query(on_query, Id, Request). + +call_batch_query(Id, Batch) -> + do_call_query(on_batch_query, Id, Batch). + +do_call_query(Fun, Id, Data) -> case emqx_resource_manager:ets_lookup(Id) of {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> - try Mod:on_query(Id, Request, ResourceState) of + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Data)), + try Mod:Fun(Id, Data, ResourceState) of + %% if the callback module (connector) wants to return an error that + %% makes the current resource goes into the `error` state, it should + %% return `{resource_down, Reason}` Result -> Result catch Err:Reason:ST -> - ModB = atom_to_binary(Mod, utf8), - Msg = <<"call failed, func: ", ModB/binary, ":on_query/3">>, - exception_error(Reason, Msg, {Err, Reason, ST}) + Msg = io_lib:format( + "call query failed, func: ~s:~s/3, error: ~0p", + [Mod, Fun, {Err, Reason, ST}] + ), + ?RESOURCE_ERROR(exception, Msg) end; {ok, _Group, #{status := stopped}} -> - resource_error(stopped, <<"resource stopped or disabled">>); + ?RESOURCE_ERROR(stopped, "resource stopped or disabled"); {ok, _Group, _Data} -> - resource_error(not_connected, <<"resource not connected">>); + ?RESOURCE_ERROR(not_connected, "resource not connected"); {error, not_found} -> - resource_error(not_found, <<"resource not found">>) + ?RESOURCE_ERROR(not_found, "resource not found") end. -call_batch_query(Id, Batch) -> - ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)), - case emqx_resource_manager:ets_lookup(Id) of - {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> - try Mod:on_batch_query(Id, Batch, ResourceState) of - BatchResults -> BatchResults - catch - Err:Reason:ST -> - ModB = atom_to_binary(Mod, utf8), - Msg = <<"call failed, func: ", ModB/binary, ":on_batch_query/3">>, - ?EXPAND(exception_error(Reason, Msg, {Err, Reason, ST}), Batch) - end; - {ok, _Group, _Data} -> - ?EXPAND(resource_error(not_connected, <<"resource not connected">>), Batch); - {error, not_found} -> - ?EXPAND(resource_error(not_found, <<"resource not found">>), Batch) - end. +%% the result is already expaned by the `Mod:on_query/3` +maybe_expand_batch_result(Results, _Batch) when is_list(Results) -> + Results; +%% the `Mod:on_query/3` returns a sinle result for a batch, so it is need expand +maybe_expand_batch_result(Result, Batch) -> + ?EXPAND(Result, Batch). -resource_error(Reason, Msg) -> - {error, {resource_error, #{reason => Reason, msg => Msg}}}. -exception_error(Reason, Msg, Details) -> - {error, {exception, #{reason => Reason, msg => Msg, details => Details}}}. +%%============================================================================== + +-spec name(id(), integer()) -> atom(). +name(Id, Index) -> + list_to_atom(lists:concat([?MODULE, ":", Id, ":", Index])). + +disk_queue_dir(Id) -> + filename:join([emqx:data_dir(), Id, queue]). -%% ========================================== ensure_flush_timer(St = #{tref := undefined, batch_time := T}) -> Ref = make_ref(), TRef = erlang:send_after(T, self(), {flush, Ref}), @@ -239,14 +326,3 @@ cancel_flush_timer(St = #{tref := undefined}) -> cancel_flush_timer(St = #{tref := {TRef, _Ref}}) -> _ = erlang:cancel_timer(TRef), St#{tref => undefined}. - -query_mfa(InsertMode, Request, SyncTimeout) -> - {?MODULE, query_fun(InsertMode), query_args(InsertMode, Request, SyncTimeout)}. - -query_fun(<<"sync">>) -> query; -query_fun(<<"async">>) -> query_async. - -query_args(<<"sync">>, Request, SyncTimeout) -> - [Request, SyncTimeout]; -query_args(<<"async">>, Request, _) -> - [Request]. From 0087b7c960a77c62f2caa873d425cafb22f0b866 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 27 Jul 2022 15:43:23 +0800 Subject: [PATCH 30/71] fix: remove the extra file replay.erl --- .../src/emqx_resource_manager.erl | 1 + .../src/emqx_resource_worker.erl | 25 +- apps/emqx_resource/src/replayq.erl | 779 ------------------ 3 files changed, 20 insertions(+), 785 deletions(-) delete mode 100644 apps/emqx_resource/src/replayq.erl diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index ee18a6836..b8a3812b5 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -423,6 +423,7 @@ handle_remove_event(From, ClearMetrics, Data) -> true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok end, + ok = emqx_resource_sup:stop_workers(Data#data.id, Data#data.opts), {stop_and_reply, normal, [{reply, From, ok}]}. start_resource(Data, From) -> diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 3c5c7eefe..7acf2d0f9 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -27,7 +27,9 @@ -export([ start_link/3, + query/2, query/3, + query_async/3, query_async/4, block/1, resume/1 @@ -70,10 +72,18 @@ callback_mode() -> [state_functions, state_enter]. start_link(Id, Index, Opts) -> gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []). +-spec query(id(), request()) -> ok. +query(Id, Request) -> + gen_statem:call(pick(Id, self()), {query, Request}). + -spec query(id(), term(), request()) -> ok. query(Id, Key, Request) -> gen_statem:call(pick(Id, Key), {query, Request}). +-spec query_async(id(), request(), reply_fun()) -> ok. +query_async(Id, Request, ReplyFun) -> + gen_statem:cast(pick(Id, self()), {query, Request, ReplyFun}). + -spec query_async(id(), term(), request(), reply_fun()) -> ok. query_async(Id, Key, Request, ReplyFun) -> gen_statem:cast(pick(Id, Key), {query, Request, ReplyFun}). @@ -91,7 +101,7 @@ init({Id, Index, Opts}) -> BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), Queue = case maps:get(queue_enabled, Opts, true) of - true -> replayq:open(#{dir => disk_queue_dir(Id), seg_bytes => 10000000}); + true -> replayq:open(#{dir => disk_queue_dir(Id, Index), seg_bytes => 10000000}); false -> undefined end, St = #{ @@ -157,13 +167,13 @@ do_resume(#{queue := undefined} = St) -> do_resume(#{queue := Q, id := Id} = St) -> case replayq:peek(Q) of empty -> - {next_state, running, St, {state_timeout, ?RESUME_INTERVAL, resume}}; + {next_state, running, St}; First -> Result = call_query(Id, First), case handle_query_result(Id, false, Result) of %% Send failed because resource down true -> - {keep_state, St}; + {keep_state, St, {state_timeout, ?RESUME_INTERVAL, resume}}; %% Send ok or failed but the resource is working false -> %% We Send 'resume' to the end of the mailbox to give the worker @@ -309,10 +319,13 @@ maybe_expand_batch_result(Result, Batch) -> -spec name(id(), integer()) -> atom(). name(Id, Index) -> - list_to_atom(lists:concat([?MODULE, ":", Id, ":", Index])). + Mod = atom_to_list(?MODULE), + Id1 = binary_to_list(Id), + Index1 = integer_to_list(Index), + list_to_atom(lists:concat([Mod, ":", Id1, ":", Index1])). -disk_queue_dir(Id) -> - filename:join([emqx:data_dir(), Id, queue]). +disk_queue_dir(Id, Index) -> + filename:join([node(), emqx:data_dir(), Id, "queue:" ++ integer_to_list(Index)]). ensure_flush_timer(St = #{tref := undefined, batch_time := T}) -> Ref = make_ref(), diff --git a/apps/emqx_resource/src/replayq.erl b/apps/emqx_resource/src/replayq.erl deleted file mode 100644 index 437702a5e..000000000 --- a/apps/emqx_resource/src/replayq.erl +++ /dev/null @@ -1,779 +0,0 @@ --module(replayq). - --export([open/1, close/1]). --export([append/2, pop/2, ack/2, ack_sync/2, peek/1, overflow/1]). --export([count/1, bytes/1, is_empty/1, is_mem_only/1]). -%% exported for troubleshooting --export([do_read_items/2]). - -%% internal exports for beam reload --export([committer_loop/2, default_sizer/1, default_marshaller/1]). - --export_type([config/0, q/0, ack_ref/0, sizer/0, marshaller/0]). - --define(NOTHING_TO_ACK, nothing_to_ack). --define(PENDING_ACKS(Ref), {replayq_pending_acks, Ref}). - --type segno() :: pos_integer(). --type item() :: term(). --type count() :: non_neg_integer(). --type id() :: count(). --type bytes() :: non_neg_integer(). --type filename() :: file:filename_all(). --type dir() :: filename(). --type ack_ref() :: ?NOTHING_TO_ACK | {segno(), ID :: pos_integer()}. --type sizer() :: fun((item()) -> bytes()). --type marshaller() :: fun((item()) -> binary()). - --type config() :: #{ - dir => dir(), - seg_bytes => bytes(), - mem_only => boolean(), - max_total_bytes => bytes(), - offload => boolean(), - sizer => sizer(), - marshaller => marshaller() -}. -%% writer cursor --define(NO_FD, no_fd). --type w_cur() :: #{ - segno := segno(), - bytes := bytes(), - count := count(), - fd := ?NO_FD | file:fd() -}. - --type stats() :: #{ - bytes := bytes(), - count := count() -}. - --opaque q() :: #{ - config := mem_only | config(), - stats := stats(), - in_mem := queue:queue(in_mem_item()), - w_cur => w_cur(), - committer => pid(), - head_segno => segno(), - sizer := sizer(), - marshaller => marshaller(), - max_total_bytes := bytes() -}. - --define(LAYOUT_VSN_0, 0). --define(LAYOUT_VSN_1, 1). --define(MAGIC, 841265288). --define(SUFFIX, "replaylog"). --define(DEFAULT_POP_BYTES_LIMIT, 2000000). --define(DEFAULT_POP_COUNT_LIMIT, 1000). --define(DEFAULT_REPLAYQ_LIMIT, 2000000000). --define(COMMIT(SEGNO, ID, From), {commit, SEGNO, ID, From}). --define(NO_COMMIT_HIST, no_commit_hist). --define(FIRST_SEGNO, 1). --define(NEXT_SEGNO(N), (N + 1)). --define(STOP, stop). --define(MEM_ONLY_ITEM(Bytes, Item), {Bytes, Item}). --define(DISK_CP_ITEM(Id, Bytes, Item), {Id, Bytes, Item}). - --type in_mem_item() :: - ?MEM_ONLY_ITEM(bytes(), item()) - | ?DISK_CP_ITEM(id(), bytes(), item()). - --spec open(config()) -> q(). -open(#{mem_only := true} = C) -> - #{ - stats => #{bytes => 0, count => 0}, - in_mem => queue:new(), - sizer => get_sizer(C), - config => mem_only, - max_total_bytes => maps:get(max_total_bytes, C, ?DEFAULT_REPLAYQ_LIMIT) - }; -open(#{dir := Dir, seg_bytes := _} = Config) -> - ok = filelib:ensure_dir(filename:join(Dir, "foo")), - Sizer = get_sizer(Config), - Marshaller = get_marshaller(Config), - IsOffload = is_offload_mode(Config), - Q = - case delete_consumed_and_list_rest(Dir) of - [] -> - %% no old segments, start over from zero - #{ - stats => #{bytes => 0, count => 0}, - w_cur => init_writer(Dir, empty, IsOffload), - committer => spawn_committer(?FIRST_SEGNO, Dir), - head_segno => ?FIRST_SEGNO, - in_mem => queue:new() - }; - Segs -> - LastSegno = lists:last(Segs), - CommitHist = get_commit_hist(Dir), - Reader = fun(Seg, Ch) -> read_items(Dir, Seg, Ch, Sizer, Marshaller) end, - HeadItems = Reader(hd(Segs), CommitHist), - #{ - stats => collect_stats(HeadItems, tl(Segs), Reader), - w_cur => init_writer(Dir, LastSegno, IsOffload), - committer => spawn_committer(hd(Segs), Dir), - head_segno => hd(Segs), - in_mem => queue:from_list(HeadItems) - } - end, - Q#{ - sizer => Sizer, - marshaller => Marshaller, - config => maps:without([sizer, marshaller], Config), - max_total_bytes => maps:get(max_total_bytes, Config, ?DEFAULT_REPLAYQ_LIMIT) - }. - --spec close(q() | w_cur()) -> ok | {error, any()}. -close(#{config := mem_only}) -> - ok; -close(#{w_cur := W_Cur, committer := Pid} = Q) -> - MRef = erlang:monitor(process, Pid), - Pid ! ?STOP, - unlink(Pid), - receive - {'DOWN', MRef, process, Pid, _Reason} -> - ok - end, - ok = maybe_dump_back_to_disk(Q), - do_close(W_Cur). - -do_close(#{fd := ?NO_FD}) -> ok; -do_close(#{fd := Fd}) -> file:close(Fd). - -%% In case of offload mode, dump the unacked (and un-popped) on disk -%% before close. this serves as a best-effort data loss protection -maybe_dump_back_to_disk(#{config := Config} = Q) -> - case is_offload_mode(Config) of - true -> dump_back_to_disk(Q); - false -> ok - end. - -dump_back_to_disk(#{ - config := #{dir := Dir}, - head_segno := ReaderSegno, - in_mem := InMem, - marshaller := Marshaller -}) -> - IoData0 = get_unacked(process_info(self(), dictionary), ReaderSegno, Marshaller), - Items1 = queue:to_list(InMem), - IoData1 = lists:map(fun(?DISK_CP_ITEM(_, _, I)) -> make_iodata(I, Marshaller) end, Items1), - %% ensure old segment file is deleted - ok = ensure_deleted(filename(Dir, ReaderSegno)), - %% rewrite the segment with what's currently in memory - IoData = [IoData0, IoData1], - case iolist_size(IoData) > 0 of - true -> - #{fd := Fd} = open_segment(Dir, ReaderSegno), - ok = file:write(Fd, [IoData0, IoData1]), - ok = file:close(Fd); - false -> - %% nothing to write - ok - end. - -get_unacked({dictionary, Dict}, ReaderSegno, Marshaller) -> - F = fun - ({?PENDING_ACKS(AckRef), Items}) -> - erase(?PENDING_ACKS(AckRef)), - {Segno, Id} = AckRef, - Segno =:= ReaderSegno andalso - {true, {Id, Items}}; - (_) -> - false - end, - Pendings0 = lists:filtermap(F, Dict), - Pendings = lists:keysort(1, Pendings0), - do_get_unacked(Pendings, Marshaller). - -do_get_unacked([], _Marshaller) -> - []; -do_get_unacked([{_, Items} | Rest], Marshaller) -> - [ - [make_iodata(I, Marshaller) || I <- Items] - | do_get_unacked(Rest, Marshaller) - ]. - --spec append(q(), [item()]) -> q(). -append(Q, []) -> - Q; -append( - #{ - config := mem_only, - in_mem := InMem, - stats := #{bytes := Bytes0, count := Count0}, - sizer := Sizer - } = Q, - Items0 -) -> - {CountDiff, BytesDiff, Items} = transform(false, Items0, Sizer), - - Stats = #{count => Count0 + CountDiff, bytes => Bytes0 + BytesDiff}, - Q#{ - stats := Stats, - in_mem := append_in_mem(Items, InMem) - }; -append( - #{ - config := #{seg_bytes := BytesLimit, dir := Dir} = Config, - stats := #{bytes := Bytes0, count := Count0}, - w_cur := #{count := CountInSeg, segno := WriterSegno} = W_Cur0, - head_segno := ReaderSegno, - sizer := Sizer, - marshaller := Marshaller, - in_mem := HeadItems0 - } = Q, - Items0 -) -> - IoData = lists:map(fun(I) -> make_iodata(I, Marshaller) end, Items0), - {CountDiff, BytesDiff, Items} = transform(CountInSeg + 1, Items0, Sizer), - TotalBytes = Bytes0 + BytesDiff, - Stats = #{count => Count0 + CountDiff, bytes => TotalBytes}, - IsOffload = is_offload_mode(Config), - W_Cur1 = do_append(W_Cur0, CountDiff, BytesDiff, IoData), - W_Cur = - case is_segment_full(W_Cur1, TotalBytes, BytesLimit, ReaderSegno, IsOffload) of - true -> - ok = do_close(W_Cur1), - %% get ready for the next append - open_segment(Dir, ?NEXT_SEGNO(WriterSegno)); - false -> - W_Cur1 - end, - HeadItems = - case ReaderSegno =:= WriterSegno of - true -> append_in_mem(Items, HeadItems0); - false -> HeadItems0 - end, - Q#{ - stats := Stats, - w_cur := W_Cur, - in_mem := HeadItems - }. - -%% @doc pop out at least one item from the queue. -%% volume limited by `bytes_limit' and `count_limit'. --spec pop(q(), #{bytes_limit => bytes(), count_limit => count()}) -> - {q(), ack_ref(), [item()]}. -pop(Q, Opts) -> - Bytes = maps:get(bytes_limit, Opts, ?DEFAULT_POP_BYTES_LIMIT), - Count = maps:get(count_limit, Opts, ?DEFAULT_POP_COUNT_LIMIT), - true = (Count > 0), - pop(Q, Bytes, Count, ?NOTHING_TO_ACK, []). - -%% @doc peek the queue front item. --spec peek(q()) -> empty | item(). -peek(#{in_mem := HeadItems}) -> - case queue:peek(HeadItems) of - empty -> empty; - {value, ?MEM_ONLY_ITEM(_, Item)} -> Item; - {value, ?DISK_CP_ITEM(_, _, Item)} -> Item - end. - -%% @doc Asynch-ly write the consumed item Segment number + ID to a file. --spec ack(q(), ack_ref()) -> ok. -ack(_, ?NOTHING_TO_ACK) -> - ok; -ack(#{committer := Pid}, {Segno, Id} = AckRef) -> - _ = erlang:erase(?PENDING_ACKS(AckRef)), - Pid ! ?COMMIT(Segno, Id, false), - ok. - -%% @hidden Synced ack, for deterministic tests only --spec ack_sync(q(), ack_ref()) -> ok. -ack_sync(_, ?NOTHING_TO_ACK) -> - ok; -ack_sync(#{committer := Pid}, {Segno, Id} = AckRef) -> - _ = erlang:erase(?PENDING_ACKS(AckRef)), - Ref = make_ref(), - Pid ! ?COMMIT(Segno, Id, {self(), Ref}), - receive - {Ref, ok} -> ok - end. - --spec count(q()) -> count(). -count(#{stats := #{count := Count}}) -> Count. - --spec bytes(q()) -> bytes(). -bytes(#{stats := #{bytes := Bytes}}) -> Bytes. - -is_empty(#{config := mem_only, in_mem := All}) -> - queue:is_empty(All); -is_empty( - #{ - w_cur := #{segno := WriterSegno}, - head_segno := ReaderSegno, - in_mem := HeadItems - } = Q -) -> - Result = ((WriterSegno =:= ReaderSegno) andalso queue:is_empty(HeadItems)), - %% assert - Result = (count(Q) =:= 0). - -%% @doc Returns number of bytes the size of the queue has exceeded -%% total bytes limit. Result is negative when it is not overflow. --spec overflow(q()) -> integer(). -overflow(#{ - max_total_bytes := MaxTotalBytes, - stats := #{bytes := Bytes} -}) -> - Bytes - MaxTotalBytes. - --spec is_mem_only(q()) -> boolean(). -is_mem_only(#{config := mem_only}) -> - true; -is_mem_only(_) -> - false. - -%% internals ========================================================= - -transform(Id, Items, Sizer) -> - transform(Id, Items, Sizer, 0, 0, []). - -transform(_Id, [], _Sizer, Count, Bytes, Acc) -> - {Count, Bytes, lists:reverse(Acc)}; -transform(Id, [Item0 | Rest], Sizer, Count, Bytes, Acc) -> - Size = Sizer(Item0), - {NextId, Item} = - case Id of - false -> {false, ?MEM_ONLY_ITEM(Size, Item0)}; - N -> {N + 1, ?DISK_CP_ITEM(Id, Size, Item0)} - end, - transform(NextId, Rest, Sizer, Count + 1, Bytes + Size, [Item | Acc]). - -append_in_mem([], Q) -> Q; -append_in_mem([Item | Rest], Q) -> append_in_mem(Rest, queue:in(Item, Q)). - -pop(Q, _Bytes, 0, AckRef, Acc) -> - Result = lists:reverse(Acc), - ok = maybe_save_pending_acks(AckRef, Q, Result), - {Q, AckRef, Result}; -pop(#{config := Cfg} = Q, Bytes, Count, AckRef, Acc) -> - case is_empty(Q) of - true -> - {Q, AckRef, lists:reverse(Acc)}; - false when Cfg =:= mem_only -> - pop_mem(Q, Bytes, Count, Acc); - false -> - pop2(Q, Bytes, Count, AckRef, Acc) - end. - -pop_mem( - #{ - in_mem := InMem, - stats := #{count := TotalCount, bytes := TotalBytes} = Stats - } = Q, - Bytes, - Count, - Acc -) -> - case queue:out(InMem) of - {{value, ?MEM_ONLY_ITEM(Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] -> - {Q, ?NOTHING_TO_ACK, lists:reverse(Acc)}; - {{value, ?MEM_ONLY_ITEM(Sz, Item)}, Rest} -> - NewQ = Q#{ - in_mem := Rest, - stats := Stats#{ - count := TotalCount - 1, - bytes := TotalBytes - Sz - } - }, - pop(NewQ, Bytes - Sz, Count - 1, ?NOTHING_TO_ACK, [Item | Acc]) - end. - -pop2( - #{ - head_segno := ReaderSegno, - in_mem := HeadItems, - stats := #{count := TotalCount, bytes := TotalBytes} = Stats, - w_cur := #{segno := WriterSegno} - } = Q, - Bytes, - Count, - AckRef, - Acc -) -> - case queue:out(HeadItems) of - {{value, ?DISK_CP_ITEM(_, Sz, _Item)}, _} when Sz > Bytes andalso Acc =/= [] -> - %% taking the head item would cause exceeding size limit - {Q, AckRef, lists:reverse(Acc)}; - {{value, ?DISK_CP_ITEM(Id, Sz, Item)}, Rest} -> - Q1 = Q#{ - in_mem := Rest, - stats := Stats#{ - count := TotalCount - 1, - bytes := TotalBytes - Sz - } - }, - %% read the next segment in case current is drained - NewQ = - case queue:is_empty(Rest) andalso ReaderSegno < WriterSegno of - true -> read_next_seg(Q1); - false -> Q1 - end, - NewAckRef = {ReaderSegno, Id}, - pop(NewQ, Bytes - Sz, Count - 1, NewAckRef, [Item | Acc]) - end. - -%% due to backward compatibility reasons for the ack api -%% we ca nnot track pending acks in q() -- reason to use process dictionary -maybe_save_pending_acks(?NOTHING_TO_ACK, _, _) -> - ok; -maybe_save_pending_acks(AckRef, #{config := Config}, Items) -> - case is_offload_mode(Config) of - true -> - _ = erlang:put(?PENDING_ACKS(AckRef), Items), - ok; - false -> - ok - end. - -read_next_seg( - #{ - config := #{dir := Dir} = Config, - head_segno := ReaderSegno, - w_cur := #{segno := WriterSegno, fd := Fd} = WCur0, - sizer := Sizer, - marshaller := Marshaller - } = Q -) -> - NextSegno = ReaderSegno + 1, - %% reader has caught up to latest segment - case NextSegno =:= WriterSegno of - true -> - %% force flush to disk so the next read can get all bytes - ok = file:sync(Fd); - false -> - ok - end, - IsOffload = is_offload_mode(Config), - WCur = - case IsOffload andalso NextSegno =:= WriterSegno of - true -> - %% reader has caught up to latest segment in offload mode, - %% close the writer's fd. Continue in mem-only mode for the head segment - ok = do_close(WCur0), - WCur0#{fd := ?NO_FD}; - false -> - WCur0 - end, - NextSegItems = read_items(Dir, NextSegno, ?NO_COMMIT_HIST, Sizer, Marshaller), - Q#{ - head_segno := NextSegno, - in_mem := queue:from_list(NextSegItems), - w_cur := WCur - }. - -delete_consumed_and_list_rest(Dir0) -> - Dir = unicode:characters_to_list(Dir0), - Segnos0 = lists:sort([parse_segno(N) || N <- filelib:wildcard("*." ?SUFFIX, Dir)]), - {SegnosToDelete, Segnos} = find_segnos_to_delete(Dir, Segnos0), - ok = lists:foreach(fun(Segno) -> ensure_deleted(filename(Dir, Segno)) end, SegnosToDelete), - case Segnos of - [] -> - %% delete commit file in case there is no segments left - %% segment number will start from 0 again. - ensure_deleted(commit_filename(Dir)), - []; - X -> - X - end. - -find_segnos_to_delete(Dir, Segnos) -> - CommitHist = get_commit_hist(Dir), - do_find_segnos_to_delete(Dir, Segnos, CommitHist). - -do_find_segnos_to_delete(_Dir, Segnos, ?NO_COMMIT_HIST) -> - {[], Segnos}; -do_find_segnos_to_delete(Dir, Segnos0, {CommittedSegno, CommittedId}) -> - {SegnosToDelete, Segnos} = lists:partition(fun(N) -> N < CommittedSegno end, Segnos0), - case - Segnos =/= [] andalso - hd(Segnos) =:= CommittedSegno andalso - is_all_consumed(Dir, CommittedSegno, CommittedId) - of - true -> - %% assert - CommittedSegno = hd(Segnos), - %% all items in the oldest segment have been consumed, - %% no need to keep this segment - {[CommittedSegno | SegnosToDelete], tl(Segnos)}; - _ -> - {SegnosToDelete, Segnos} - end. - -%% ALL items are consumed if the committed item ID is no-less than the number -%% of items in this segment -is_all_consumed(Dir, CommittedSegno, CommittedId) -> - CommittedId >= erlang:length(do_read_items(Dir, CommittedSegno)). - -ensure_deleted(Filename) -> - case file:delete(Filename) of - ok -> ok; - {error, enoent} -> ok - end. - -%% The committer writes consumer's acked segmeng number + item ID -%% to a file. The file is only read at start/restart. -spawn_committer(ReaderSegno, Dir) -> - Name = iolist_to_binary(filename:join([Dir, committer])), - %% register a name to avoid having two committers spawned for the same dir - RegName = binary_to_atom(Name, utf8), - Pid = erlang:spawn_link(fun() -> committer_loop(ReaderSegno, Dir) end), - true = erlang:register(RegName, Pid), - Pid. - -committer_loop(ReaderSegno, Dir) -> - receive - ?COMMIT(Segno0, Id0, false) -> - {Segno, Id} = collect_async_commits(Segno0, Id0), - ok = handle_commit(ReaderSegno, Dir, Segno, Id, false), - ?MODULE:committer_loop(Segno, Dir); - ?COMMIT(Segno, Id, From) -> - ok = handle_commit(ReaderSegno, Dir, Segno, Id, From), - ?MODULE:committer_loop(Segno, Dir); - ?STOP -> - ok; - Msg -> - exit({replayq_committer_unkown_msg, Msg}) - after 200 -> - ?MODULE:committer_loop(ReaderSegno, Dir) - end. - -handle_commit(ReaderSegno, Dir, Segno, Id, From) -> - IoData = io_lib:format("~p.\n", [#{segno => Segno, id => Id}]), - ok = do_commit(Dir, IoData), - case Segno > ReaderSegno of - true -> - SegnosToDelete = lists:seq(ReaderSegno, Segno - 1), - lists:foreach(fun(N) -> ok = ensure_deleted(filename(Dir, N)) end, SegnosToDelete); - false -> - ok - end, - ok = reply_ack_ok(From). - -%% Collect async acks which are already sent in the mailbox, -%% and keep only the last one for the current segment. -collect_async_commits(Segno, Id) -> - receive - ?COMMIT(Segno, AnotherId, false) -> - collect_async_commits(Segno, AnotherId) - after 0 -> - {Segno, Id} - end. - -reply_ack_ok({Pid, Ref}) -> - Pid ! {Ref, ok}, - ok; -reply_ack_ok(_) -> - ok. - -get_commit_hist(Dir) -> - CommitFile = commit_filename(Dir), - case filelib:is_regular(CommitFile) of - true -> - {ok, [#{segno := Segno, id := Id}]} = file:consult(CommitFile), - {Segno, Id}; - false -> - ?NO_COMMIT_HIST - end. - -do_commit(Dir, IoData) -> - TmpName = commit_filename(Dir, "COMMIT.tmp"), - Name = commit_filename(Dir), - ok = file:write_file(TmpName, IoData), - ok = file:rename(TmpName, Name). - -commit_filename(Dir) -> - commit_filename(Dir, "COMMIT"). - -commit_filename(Dir, Name) -> - filename:join([Dir, Name]). - -do_append( - #{fd := ?NO_FD, bytes := Bytes0, count := Count0} = Cur, - Count, - Bytes, - _IoData -) -> - %% offload mode, fd is not initialized yet - Cur#{ - bytes => Bytes0 + Bytes, - count => Count0 + Count - }; -do_append( - #{fd := Fd, bytes := Bytes0, count := Count0} = Cur, - Count, - Bytes, - IoData -) -> - ok = file:write(Fd, IoData), - Cur#{ - bytes => Bytes0 + Bytes, - count => Count0 + Count - }. - -read_items(Dir, Segno, CommitHist, Sizer, Marshaller) -> - Items0 = do_read_items(Dir, Segno), - Items = - case CommitHist of - ?NO_COMMIT_HIST -> - %% no commit hist, return all - Items0; - {CommitedSegno, _} when CommitedSegno < Segno -> - %% committed at an older segment - Items0; - {Segno, CommittedId} -> - %% committed at current segment keep only the tail - {_, R} = lists:splitwith(fun({I, _}) -> I =< CommittedId end, Items0), - R - end, - lists:map( - fun({Id, Bin}) -> - Item = Marshaller(Bin), - Size = Sizer(Item), - ?DISK_CP_ITEM(Id, Size, Item) - end, - Items - ). - -do_read_items(Dir, Segno) -> - Filename = filename(Dir, Segno), - {ok, Bin} = file:read_file(Filename), - case parse_items(Bin, 1, []) of - {Items, <<>>} -> - Items; - {Items, Corrupted} -> - error_logger:error_msg( - "corrupted replayq log: ~s, skipped ~p bytes", - [Filename, size(Corrupted)] - ), - Items - end. - -parse_items(<<>>, _Id, Acc) -> - {lists:reverse(Acc), <<>>}; -parse_items( - <> = All, - Id, - Acc -) -> - case CRC =:= erlang:crc32(Item) of - true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]); - false -> {lists:reverse(Acc), All} - end; -parse_items( - <> = All, - Id, - Acc -) -> - case CRC =:= erlang:crc32(Item) andalso Item =/= <<>> of - true -> parse_items(Rest, Id + 1, [{Id, Item} | Acc]); - false -> {lists:reverse(Acc), All} - end; -parse_items(Corrupted, _Id, Acc) -> - {lists:reverse(Acc), Corrupted}. - -make_iodata(Item0, Marshaller) -> - Item = Marshaller(Item0), - Size = size(Item), - CRC = erlang:crc32(Item), - [ - <>, - Item - ]. - -collect_stats(HeadItems, SegsOnDisk, Reader) -> - ItemF = fun(?DISK_CP_ITEM(_Id, Sz, _Item), {B, C}) -> - {B + Sz, C + 1} - end, - Acc0 = lists:foldl(ItemF, {0, 0}, HeadItems), - {Bytes, Count} = - lists:foldl( - fun(Segno, Acc) -> - Items = Reader(Segno, ?NO_COMMIT_HIST), - lists:foldl(ItemF, Acc, Items) - end, - Acc0, - SegsOnDisk - ), - #{bytes => Bytes, count => Count}. - -parse_segno(Filename) -> - [Segno, ?SUFFIX] = string:tokens(Filename, "."), - list_to_integer(Segno). - -filename(Dir, Segno) -> - Name = lists:flatten(io_lib:format("~10.10.0w." ?SUFFIX, [Segno])), - filename:join(Dir, Name). - -%% open the current segment for write if it is empty -%% otherwise rollout to the next segment --spec init_writer(dir(), empty | segno(), boolean()) -> w_cur(). -init_writer(_Dir, empty, true) -> - %% clean start for offload mode - #{fd => ?NO_FD, segno => ?FIRST_SEGNO, bytes => 0, count => 0}; -init_writer(Dir, empty, false) -> - open_segment(Dir, ?FIRST_SEGNO); -init_writer(Dir, Segno, _IsOffload) when is_number(Segno) -> - Filename = filename(Dir, Segno), - case filelib:file_size(Filename) of - 0 -> open_segment(Dir, Segno); - _ -> open_segment(Dir, ?NEXT_SEGNO(Segno)) - end. - --spec open_segment(dir(), segno()) -> w_cur(). -open_segment(Dir, Segno) -> - Filename = filename(Dir, Segno), - %% raw so there is no need to go through the single gen_server file_server - {ok, Fd} = file:open(Filename, [raw, read, write, binary, delayed_write]), - #{fd => Fd, segno => Segno, bytes => 0, count => 0}. - -get_sizer(C) -> - maps:get(sizer, C, fun ?MODULE:default_sizer/1). - -get_marshaller(C) -> - maps:get(marshaller, C, fun ?MODULE:default_marshaller/1). - -is_offload_mode(Config) when is_map(Config) -> - maps:get(offload, Config, false). - -default_sizer(I) when is_binary(I) -> erlang:size(I). - -default_marshaller(I) when is_binary(I) -> I. - -is_segment_full( - #{segno := WriterSegno, bytes := SegmentBytes}, - TotalBytes, - SegmentBytesLimit, - ReaderSegno, - true -) -> - %% in offload mode, when reader is lagging behind, we try - %% writer rolls to a new segment when file size limit is reached - %% when reader is reading off from the same segment as writer - %% i.e. the in memory queue, only start writing to segment file - %% when total bytes (in memory) is reached segment limit - %% - %% NOTE: we never shrink segment bytes, even when popping out - %% from the in-memory queue. - case ReaderSegno < WriterSegno of - true -> SegmentBytes >= SegmentBytesLimit; - false -> TotalBytes >= SegmentBytesLimit - end; -is_segment_full( - #{bytes := SegmentBytes}, - _TotalBytes, - SegmentBytesLimit, - _ReaderSegno, - false -) -> - %% here we check if segment size is greater than segment size limit - %% after append based on the assumption that the items are usually - %% very small in size comparing to segment size. - %% We can change implementation to split items list to avoid - %% segment overflow if really necessary - SegmentBytes >= SegmentBytesLimit. From 2fb42e4d37442b141e1b0259bc71ae5807502704 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 28 Jul 2022 17:40:04 +0800 Subject: [PATCH 31/71] refactor: create emqx_resource_worker_sup for resource workers --- .../emqx_authn_jwks_connector.erl | 12 +- .../src/simple_authn/emqx_authn_mongodb.erl | 2 +- .../src/emqx_connector_http.erl | 29 ++-- .../src/emqx_connector_ldap.erl | 9 +- .../src/emqx_connector_mongo.erl | 10 +- .../src/emqx_connector_mqtt.erl | 10 +- .../src/emqx_connector_mysql.erl | 17 +-- .../src/emqx_connector_pgsql.erl | 13 +- .../src/emqx_connector_redis.erl | 9 +- apps/emqx_resource/include/emqx_resource.hrl | 14 +- apps/emqx_resource/src/emqx_resource.erl | 32 +---- .../src/emqx_resource_manager.erl | 15 +- apps/emqx_resource/src/emqx_resource_sup.erl | 85 ++--------- .../src/emqx_resource_worker.erl | 132 ++++++++++------- .../src/emqx_resource_worker_sup.erl | 136 ++++++++++++++++++ .../test/emqx_resource_SUITE.erl | 77 +++++----- .../emqx_resource/test/emqx_test_resource.erl | 27 ++-- 17 files changed, 332 insertions(+), 297 deletions(-) create mode 100644 apps/emqx_resource/src/emqx_resource_worker_sup.erl diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index 8f98e2f1e..cd8451ac9 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -24,7 +24,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2, connect/1 ]). @@ -45,7 +45,7 @@ on_start(InstId, Opts) -> on_stop(_InstId, #{pool_name := PoolName}) -> emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, get_jwks, AfterQuery, #{pool_name := PoolName}) -> +on_query(InstId, get_jwks, #{pool_name := PoolName}) -> Result = ecpool:pick_and_do(PoolName, {emqx_authn_jwks_client, get_jwks, []}, no_handover), case Result of {error, Reason} -> @@ -54,20 +54,18 @@ on_query(InstId, get_jwks, AfterQuery, #{pool_name := PoolName}) -> connector => InstId, command => get_jwks, reason => Reason - }), - emqx_resource:query_failed(AfterQuery); + }); _ -> - emqx_resource:query_success(AfterQuery) + ok end, Result; -on_query(_InstId, {update, Opts}, AfterQuery, #{pool_name := PoolName}) -> +on_query(_InstId, {update, Opts}, #{pool_name := PoolName}) -> lists:foreach( fun({_, Worker}) -> ok = ecpool_worker:exec(Worker, {emqx_authn_jwks_client, update, [Opts]}, infinity) end, ecpool:workers(PoolName) ), - emqx_resource:query_success(AfterQuery), ok. on_get_status(_InstId, #{pool_name := PoolName}) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index f7249ae57..ff9c97717 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -174,7 +174,7 @@ authenticate( reason => Reason }), ignore; - Doc -> + {ok, Doc} -> case check_password(Password, Doc, State) of ok -> {ok, is_superuser(Doc, State)}; diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 59b4ddffa..e0d5ccfe0 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -28,7 +28,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -225,7 +225,7 @@ on_stop(InstId, #{pool_name := PoolName}) -> }), ehttpc_sup:stop_pool(PoolName). -on_query(InstId, {send_message, Msg}, AfterQuery, State) -> +on_query(InstId, {send_message, Msg}, State) -> case maps:get(request, State, undefined) of undefined -> ?SLOG(error, #{msg => "request_not_found", connector => InstId}); @@ -241,18 +241,16 @@ on_query(InstId, {send_message, Msg}, AfterQuery, State) -> on_query( InstId, {undefined, Method, {Path, Headers, Body}, Timeout, Retry}, - AfterQuery, State ) end; -on_query(InstId, {Method, Request}, AfterQuery, State) -> - on_query(InstId, {undefined, Method, Request, 5000, 2}, AfterQuery, State); -on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> - on_query(InstId, {undefined, Method, Request, Timeout, 2}, AfterQuery, State); +on_query(InstId, {Method, Request}, State) -> + on_query(InstId, {undefined, Method, Request, 5000, 2}, State); +on_query(InstId, {Method, Request, Timeout}, State) -> + on_query(InstId, {undefined, Method, Request, Timeout, 2}, State); on_query( InstId, {KeyOrNum, Method, Request, Timeout, Retry}, - AfterQuery, #{pool_name := PoolName, base_path := BasePath} = State ) -> ?TRACE( @@ -275,32 +273,29 @@ on_query( of {error, Reason} -> ?SLOG(error, #{ - msg => "http_connector_do_reqeust_failed", + msg => "http_connector_do_request_failed", request => NRequest, reason => Reason, connector => InstId - }), - emqx_resource:query_failed(AfterQuery); + }); {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 -> - emqx_resource:query_success(AfterQuery); + ok; {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 -> - emqx_resource:query_success(AfterQuery); + ok; {ok, StatusCode, _} -> ?SLOG(error, #{ msg => "http connector do request, received error response", request => NRequest, connector => InstId, status_code => StatusCode - }), - emqx_resource:query_failed(AfterQuery); + }); {ok, StatusCode, _, _} -> ?SLOG(error, #{ msg => "http connector do request, received error response", request => NRequest, connector => InstId, status_code => StatusCode - }), - emqx_resource:query_failed(AfterQuery) + }) end, Result. diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 195aa89a9..51d18b534 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -27,7 +27,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -99,7 +99,7 @@ on_stop(InstId, #{poolname := PoolName}) -> }), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) -> +on_query(InstId, {search, Base, Filter, Attributes}, #{poolname := PoolName} = State) -> Request = {Base, Filter, Attributes}, ?TRACE( "QUERY", @@ -119,10 +119,9 @@ on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := P request => Request, connector => InstId, reason => Reason - }), - emqx_resource:query_failed(AfterQuery); + }); _ -> - emqx_resource:query_success(AfterQuery) + ok end, Result. diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 5b07c5003..db8b1e632 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -27,7 +27,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -189,7 +189,6 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query( InstId, {Action, Collection, Filter, Projector}, - AfterQuery, #{poolname := PoolName} = State ) -> Request = {Action, Collection, Filter, Projector}, @@ -212,14 +211,11 @@ on_query( reason => Reason, connector => InstId }), - emqx_resource:query_failed(AfterQuery), {error, Reason}; {ok, Cursor} when is_pid(Cursor) -> - emqx_resource:query_success(AfterQuery), - mc_cursor:foldl(fun(O, Acc2) -> [O | Acc2] end, [], Cursor, 1000); + {ok, mc_cursor:foldl(fun(O, Acc2) -> [O | Acc2] end, [], Cursor, 1000)}; Result -> - emqx_resource:query_success(AfterQuery), - Result + {ok, Result} end. -dialyzer({nowarn_function, [on_get_status/2]}). diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 21e201504..98635de3f 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -37,7 +37,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -181,12 +181,12 @@ on_stop(_InstId, #{name := InstanceId}) -> }) end. -on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) -> - emqx_resource:query_success(AfterQuery); -on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) -> +on_query(_InstId, {message_received, _Msg}, _State) -> + ok; +on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) -> ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg), - emqx_resource:query_success(AfterQuery). + ok. on_get_status(_InstId, #{name := InstanceId, bridge_conf := Conf}) -> AutoReconn = maps:get(auto_reconnect, Conf, true), diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 409da4060..e818bc6ef 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -26,7 +26,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -122,14 +122,13 @@ on_stop(InstId, #{poolname := PoolName}) -> }), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {TypeOrKey, SQLOrKey}, AfterQuery, State) -> - on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, AfterQuery, State); -on_query(InstId, {TypeOrKey, SQLOrKey, Params}, AfterQuery, State) -> - on_query(InstId, {TypeOrKey, SQLOrKey, Params, default_timeout}, AfterQuery, State); +on_query(InstId, {TypeOrKey, SQLOrKey}, State) -> + on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, State); +on_query(InstId, {TypeOrKey, SQLOrKey, Params}, State) -> + on_query(InstId, {TypeOrKey, SQLOrKey, Params, default_timeout}, State); on_query( InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, - AfterQuery, #{poolname := PoolName, prepare_statement := Prepares} = State ) -> LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, @@ -147,7 +146,6 @@ on_query( ), %% kill the poll worker to trigger reconnection _ = exit(Conn, restart), - emqx_resource:query_failed(AfterQuery), Result; {error, not_prepared} -> ?SLOG( @@ -157,13 +155,12 @@ on_query( case prepare_sql(Prepares, PoolName) of ok -> %% not return result, next loop will try again - on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, AfterQuery, State); + on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, State); {error, Reason} -> ?SLOG( error, LogMeta#{msg => "mysql_connector_do_prepare_failed", reason => Reason} ), - emqx_resource:query_failed(AfterQuery), {error, Reason} end; {error, Reason} -> @@ -171,10 +168,8 @@ on_query( error, LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason} ), - emqx_resource:query_failed(AfterQuery), Result; _ -> - emqx_resource:query_success(AfterQuery), Result end. diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 6f89e7ff1..d31c1316f 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -29,7 +29,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -116,9 +116,9 @@ on_stop(InstId, #{poolname := PoolName}) -> }), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {Type, NameOrSQL}, AfterQuery, #{poolname := _PoolName} = State) -> - on_query(InstId, {Type, NameOrSQL, []}, AfterQuery, State); -on_query(InstId, {Type, NameOrSQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> +on_query(InstId, {Type, NameOrSQL}, #{poolname := _PoolName} = State) -> + on_query(InstId, {Type, NameOrSQL, []}, State); +on_query(InstId, {Type, NameOrSQL, Params}, #{poolname := PoolName} = State) -> ?SLOG(debug, #{ msg => "postgresql connector received sql query", connector => InstId, @@ -132,10 +132,9 @@ on_query(InstId, {Type, NameOrSQL, Params}, AfterQuery, #{poolname := PoolName} connector => InstId, sql => NameOrSQL, reason => Reason - }), - emqx_resource:query_failed(AfterQuery); + }); _ -> - emqx_resource:query_success(AfterQuery) + ok end, Result. diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 67310dbac..4826a170b 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -28,7 +28,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -177,7 +177,7 @@ on_stop(InstId, #{poolname := PoolName, type := Type}) -> _ -> emqx_plugin_libs_pool:stop_pool(PoolName) end. -on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) -> +on_query(InstId, {cmd, Command}, #{poolname := PoolName, type := Type} = State) -> ?TRACE( "QUERY", "redis_connector_received", @@ -195,10 +195,9 @@ on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := T connector => InstId, sql => Command, reason => Reason - }), - emqx_resource:query_failed(AfterCommand); + }); _ -> - emqx_resource:query_success(AfterCommand) + ok end, Result. diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index d6f959510..13ffff587 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -21,7 +21,7 @@ -type resource_config() :: term(). -type resource_spec() :: map(). -type resource_state() :: term(). --type resource_status() :: connected | disconnected | connecting. +-type resource_status() :: connected | disconnected | connecting | stopped. -type resource_data() :: #{ id := resource_id(), mod := module(), @@ -45,13 +45,11 @@ %% periodically. auto_retry_interval => integer() }. --type after_query() :: - {[OnSuccess :: after_query_fun()], [OnFailed :: after_query_fun()]} - | undefined. - -%% the `after_query_fun()` is mainly for callbacks that increment counters or do some fallback -%% actions upon query failure --type after_query_fun() :: {fun((...) -> ok), Args :: [term()]}. +-type query_result() :: + ok + | {ok, term()} + | {error, term()} + | {resource_down, term()}. -define(TEST_ID_PREFIX, "_test_:"). -define(RES_METRICS, resource_metrics). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 793b9f446..081264315 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -23,13 +23,6 @@ -export([list_types/0]). -%% APIs for behaviour implementations - --export([ - query_success/1, - query_failed/1 -]). - %% APIs for instances -export([ @@ -113,7 +106,8 @@ -export([inc_metrics_funcs/1, inc_success/1, inc_failed/1]). -optional_callbacks([ - on_query/4, + on_query/3, + on_batch_query/3, on_get_status/2 ]). @@ -125,7 +119,9 @@ -callback on_stop(resource_id(), resource_state()) -> term(). %% when calling emqx_resource:query/3 --callback on_query(resource_id(), Request :: term(), after_query(), resource_state()) -> term(). +-callback on_query(resource_id(), Request :: term(), resource_state()) -> query_result(). + +-callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result(). %% when calling emqx_resource:health_check/2 -callback on_get_status(resource_id(), resource_state()) -> @@ -149,22 +145,6 @@ is_resource_mod(Module) -> proplists:get_value(behaviour, Info, []), lists:member(?MODULE, Behaviour). --spec query_success(after_query()) -> ok. -query_success(undefined) -> ok; -query_success({OnSucc, _}) -> exec_query_after_calls(OnSucc). - --spec query_failed(after_query()) -> ok. -query_failed(undefined) -> ok; -query_failed({_, OnFailed}) -> exec_query_after_calls(OnFailed). - -exec_query_after_calls(Funcs) -> - lists:foreach( - fun({Fun, Arg}) -> - emqx_resource_utils:safe_exec(Fun, Arg) - end, - Funcs - ). - %% ================================================================================= %% APIs for resource instances %% ================================================================================= @@ -247,7 +227,7 @@ query(ResId, Request) -> emqx_resource_worker:query(ResId, Request). -spec query_async(resource_id(), Request :: term(), emqx_resource_worker:reply_fun()) -> - ok. + Result :: term(). query_async(ResId, Request, ReplyFun) -> emqx_resource_worker:query_async(ResId, Request, ReplyFun). diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index b8a3812b5..b5bcbd330 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -53,7 +53,7 @@ -define(SHORT_HEALTHCHECK_INTERVAL, 1000). -define(HEALTHCHECK_INTERVAL, 15000). --define(ETS_TABLE, emqx_resource_manager). +-define(ETS_TABLE, ?MODULE). -define(WAIT_FOR_RESOURCE_DELAY, 100). -define(T_OPERATION, 5000). -define(T_LOOKUP, 1000). @@ -114,9 +114,9 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) -> [matched, success, failed, exception, resource_down], [matched] ), + ok = emqx_resource_worker_sup:start_workers(ResId, Opts), case maps:get(start_after_created, Opts, true) of true -> - ok = emqx_resource_sup:start_workers(ResId, Opts), wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)); false -> ok @@ -317,7 +317,7 @@ handle_event({call, From}, health_check, _State, Data) -> handle_manually_health_check(From, Data); % State: CONNECTING handle_event(enter, _OldState, connecting, Data) -> - UpdatedData = Data#data{status = connected}, + UpdatedData = Data#data{status = connecting}, insert_cache(Data#data.id, Data#data.group, Data), Actions = [{state_timeout, 0, health_check}], {keep_state, UpdatedData, Actions}; @@ -332,7 +332,7 @@ handle_event(enter, _OldState, connected, Data) -> UpdatedData = Data#data{status = connected}, insert_cache(Data#data.id, Data#data.group, UpdatedData), _ = emqx_alarm:deactivate(Data#data.id), - Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}], + Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}], {next_state, connected, UpdatedData, Actions}; handle_event(state_timeout, health_check, connected, Data) -> handle_connected_health_check(Data); @@ -423,7 +423,7 @@ handle_remove_event(From, ClearMetrics, Data) -> true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok end, - ok = emqx_resource_sup:stop_workers(Data#data.id, Data#data.opts), + ok = emqx_resource_worker_sup:stop_workers(Data#data.id, Data#data.opts), {stop_and_reply, normal, [{reply, From, ok}]}. start_resource(Data, From) -> @@ -487,7 +487,7 @@ handle_connected_health_check(Data) -> Data, fun (connected, UpdatedData) -> - Actions = [{state_timeout, ?HEALTHCHECK_INTERVAL, health_check}], + Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}], {keep_state, UpdatedData, Actions}; (Status, UpdatedData) -> ?SLOG(error, #{ @@ -510,6 +510,9 @@ with_health_check(Data, Func) -> insert_cache(ResId, UpdatedData#data.group, UpdatedData), Func(Status, UpdatedData). +health_check_interval(Opts) -> + maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). + maybe_alarm(connected, _ResId) -> ok; maybe_alarm(_Status, <>) -> diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index 14db50d01..920743101 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -19,7 +19,7 @@ -behaviour(supervisor). --export([start_link/0, start_workers/2, stop_workers/2]). +-export([start_link/0]). -export([init/1]). @@ -29,7 +29,6 @@ start_link() -> init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, Metrics = emqx_metrics_worker:child_spec(?RES_METRICS), - ResourceManager = #{ id => emqx_resource_manager_sup, @@ -39,79 +38,11 @@ init([]) -> type => supervisor, modules => [emqx_resource_manager_sup] }, - {ok, {SupFlags, [Metrics, ResourceManager]}}. - -start_workers(ResId, Opts) -> - PoolSize = pool_size(Opts), - _ = ensure_worker_pool(ResId, hash, [{size, PoolSize}]), - lists:foreach( - fun(Idx) -> - _ = ensure_worker_added(ResId, {ResId, Idx}, Idx), - ok = ensure_worker_started(ResId, Idx, Opts) - end, - lists:seq(1, PoolSize) - ). - -stop_workers(ResId, Opts) -> - PoolSize = pool_size(Opts), - lists:foreach( - fun(Idx) -> - ok = ensure_worker_stopped(ResId, Idx), - ok = ensure_worker_removed(ResId, {ResId, Idx}) - end, - lists:seq(1, PoolSize) - ), - _ = gproc_pool:delete(ResId), - ok. - -pool_size(Opts) -> - maps:get(worker_pool_size, Opts, erlang:system_info(schedulers_online)). - -ensure_worker_pool(Pool, Type, Opts) -> - try - gproc_pool:new(Pool, Type, Opts) - catch - error:exists -> ok - end, - ok. - -ensure_worker_added(Pool, Name, Slot) -> - try - gproc_pool:add_worker(Pool, Name, Slot) - catch - error:exists -> ok - end, - ok. - -ensure_worker_removed(Pool, Name) -> - _ = gproc_pool:remove_worker(Pool, Name), - ok. - --define(CHILD_ID(MOD, RESID, INDEX), {MOD, RESID, INDEX}). -ensure_worker_started(ResId, Idx, Opts) -> - Mod = emqx_resource_worker, - Spec = #{ - id => ?CHILD_ID(Mod, ResId, Idx), - start => {Mod, start_link, [ResId, Idx, Opts]}, - restart => transient, - shutdown => 5000, - type => worker, - modules => [Mod] + WorkerSup = #{ + id => emqx_resource_worker_sup, + start => {emqx_resource_worker_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor }, - case supervisor:start_child(emqx_resource_sup, Spec) of - {ok, _Pid} -> ok; - {error, {already_started, _}} -> ok; - {error, already_present} -> ok; - {error, _} = Err -> Err - end. - -ensure_worker_stopped(ResId, Idx) -> - ChildId = ?CHILD_ID(emqx_resource_worker, ResId, Idx), - case supervisor:terminate_child(emqx_resource_sup, ChildId) of - ok -> - supervisor:delete_child(emqx_resource_sup, ChildId); - {error, not_found} -> - ok; - {error, Reason} -> - {error, Reason} - end. + {ok, {SupFlags, [Metrics, ResourceManager, WorkerSup]}}. diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 7acf2d0f9..ae0d24313 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -44,6 +44,8 @@ -export([running/3, blocked/3]). +-export([queue_item_marshaller/1]). + -define(RESUME_INTERVAL, 15000). %% count @@ -51,11 +53,15 @@ %% milliseconds -define(DEFAULT_BATCH_TIME, 10). +-define(Q_ITEM(REQUEST), {q_item, REQUEST}). + -define(QUERY(FROM, REQUEST), {FROM, REQUEST}). -define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}). -define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]). --define(RESOURCE_ERROR(Reason, Msg), {error, {resource_error, #{reason => Reason, msg => Msg}}}). +-define(RESOURCE_ERROR(Reason, Msg), + {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}} +). -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}). -type id() :: binary(). @@ -72,21 +78,21 @@ callback_mode() -> [state_functions, state_enter]. start_link(Id, Index, Opts) -> gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []). --spec query(id(), request()) -> ok. +-spec query(id(), request()) -> Result :: term(). query(Id, Request) -> - gen_statem:call(pick(Id, self()), {query, Request}). + query(Id, self(), Request). --spec query(id(), term(), request()) -> ok. -query(Id, Key, Request) -> - gen_statem:call(pick(Id, Key), {query, Request}). +-spec query(id(), term(), request()) -> Result :: term(). +query(Id, PickKey, Request) -> + pick_query(call, Id, PickKey, {query, Request}). --spec query_async(id(), request(), reply_fun()) -> ok. +-spec query_async(id(), request(), reply_fun()) -> Result :: term(). query_async(Id, Request, ReplyFun) -> - gen_statem:cast(pick(Id, self()), {query, Request, ReplyFun}). + query_async(Id, self(), Request, ReplyFun). --spec query_async(id(), term(), request(), reply_fun()) -> ok. -query_async(Id, Key, Request, ReplyFun) -> - gen_statem:cast(pick(Id, Key), {query, Request, ReplyFun}). +-spec query_async(id(), term(), request(), reply_fun()) -> Result :: term(). +query_async(Id, PickKey, Request, ReplyFun) -> + pick_query(cast, Id, PickKey, {query, Request, ReplyFun}). -spec block(pid() | atom()) -> ok. block(ServerRef) -> @@ -97,17 +103,24 @@ resume(ServerRef) -> gen_statem:cast(ServerRef, resume). init({Id, Index, Opts}) -> + process_flag(trap_exit, true), true = gproc_pool:connect_worker(Id, {Id, Index}), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), Queue = case maps:get(queue_enabled, Opts, true) of - true -> replayq:open(#{dir => disk_queue_dir(Id, Index), seg_bytes => 10000000}); - false -> undefined + true -> + replayq:open(#{ + dir => disk_queue_dir(Id, Index), + seg_bytes => 10000000, + marshaller => fun ?MODULE:queue_item_marshaller/1 + }); + false -> + undefined end, St = #{ id => Id, index => Index, - batch_enabled => maps:get(batch_enabled, Opts, true), + batch_enabled => maps:get(batch_enabled, Opts, false), batch_size => BatchSize, batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), queue => Queue, @@ -128,7 +141,7 @@ running(cast, {query, Request, ReplyFun}, St) -> running({call, From}, {query, Request}, St) -> query_or_acc(From, Request, St); running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> - {keep_state, flush(St#{tref := undefined})}; + flush(St#{tref := undefined}); running(info, {flush, _Ref}, _St) -> keep_state_and_data; running(info, Info, _St) -> @@ -154,12 +167,21 @@ terminate(_Reason, #{id := Id, index := Index}) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +queue_item_marshaller(?Q_ITEM(_) = I) -> + term_to_binary(I); +queue_item_marshaller(Bin) when is_binary(Bin) -> + binary_to_term(Bin). + %%============================================================================== -pick(Id, Key) -> - Pid = gproc_pool:pick_worker(Id, Key), - case is_pid(Pid) of - true -> Pid; - false -> error({failed_to_pick_worker, {Id, Key}}) +pick_query(Fun, Id, Key, Query) -> + try gproc_pool:pick_worker(Id, Key) of + Pid when is_pid(Pid) -> + gen_statem:Fun(Pid, Query); + _ -> + ?RESOURCE_ERROR(not_created, "resource not found") + catch + error:badarg -> + ?RESOURCE_ERROR(not_created, "resource not found") end. do_resume(#{queue := undefined} = St) -> @@ -168,9 +190,9 @@ do_resume(#{queue := Q, id := Id} = St) -> case replayq:peek(Q) of empty -> {next_state, running, St}; - First -> + ?Q_ITEM(First) -> Result = call_query(Id, First), - case handle_query_result(Id, false, Result) of + case handle_query_result(Id, Result, false) of %% Send failed because resource down true -> {keep_state, St, {state_timeout, ?RESUME_INTERVAL, resume}}; @@ -182,6 +204,11 @@ do_resume(#{queue := Q, id := Id} = St) -> end end. +handle_blocked(From, Request, #{id := Id, queue := Q} = St) -> + Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), + _ = reply_caller(Id, ?REPLY(From, Request, Error), false), + {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}}. + drop_head(Q) -> {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}), ok = replayq:ack(Q1, AckRef), @@ -196,26 +223,21 @@ acc_query(From, Request, #{acc := Acc, acc_left := Left} = St0) -> Acc1 = [?QUERY(From, Request) | Acc], St = St0#{acc := Acc1, acc_left := Left - 1}, case Left =< 1 of - true -> {keep_state, flush(St)}; + true -> flush(St); false -> {keep_state, ensure_flush_timer(St)} end. send_query(From, Request, #{id := Id, queue := Q} = St) -> Result = call_query(Id, Request), - case reply_caller(Id, Q, ?REPLY(From, Request, Result)) of + case reply_caller(Id, ?REPLY(From, Request, Result), false) of true -> - {keep_state, St#{queue := maybe_append_queue(Q, [Request])}}; + {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}}; false -> - {next_state, blocked, St} + {keep_state, St} end. -handle_blocked(From, Request, #{id := Id, queue := Q} = St) -> - Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), - _ = reply_caller(Id, Q, ?REPLY(From, Request, Error)), - {keep_state, St#{queue := maybe_append_queue(Q, [Request])}}. - flush(#{acc := []} = St) -> - St; + {keep_state, St}; flush( #{ id := Id, @@ -228,65 +250,65 @@ flush( St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}), case batch_reply_caller(Id, BatchResults) of true -> - Q1 = maybe_append_queue(Q0, [Request || ?QUERY(_, Request) <- Batch]), - {keep_state, St1#{queue := Q1}}; + Q1 = maybe_append_queue(Q0, [?Q_ITEM(Request) || ?QUERY(_, Request) <- Batch]), + {next_state, blocked, St1#{queue := Q1}}; false -> - {next_state, blocked, St1} + {keep_state, St1} end. -maybe_append_queue(undefined, _Query) -> undefined; -maybe_append_queue(Q, Query) -> replayq:append(Q, Query). +maybe_append_queue(undefined, _Request) -> undefined; +maybe_append_queue(Q, Request) -> replayq:append(Q, Request). batch_reply_caller(Id, BatchResults) -> lists:foldl( fun(Reply, BlockWorker) -> - reply_caller(Id, BlockWorker, Reply) + reply_caller(Id, Reply, BlockWorker) end, false, BatchResults ). -reply_caller(Id, BlockWorker, ?REPLY(undefined, _, Result)) -> - handle_query_result(Id, BlockWorker, Result); -reply_caller(Id, BlockWorker, ?REPLY({ReplyFun, Args}, _, Result)) -> +reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) -> + handle_query_result(Id, Result, BlockWorker); +reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) -> ?SAFE_CALL(ReplyFun(Result, Args)), - handle_query_result(Id, BlockWorker, Result); -reply_caller(Id, BlockWorker, ?REPLY(From, _, Result)) -> + handle_query_result(Id, Result, BlockWorker); +reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) -> gen_statem:reply(From, Result), - handle_query_result(Id, BlockWorker, Result). + handle_query_result(Id, Result, BlockWorker). -handle_query_result(Id, BlockWorker, ok) -> +handle_query_result(Id, ok, BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, success), BlockWorker; -handle_query_result(Id, BlockWorker, {ok, _}) -> +handle_query_result(Id, {ok, _}, BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, success), BlockWorker; -handle_query_result(Id, BlockWorker, ?RESOURCE_ERROR_M(exception, _)) -> +handle_query_result(Id, ?RESOURCE_ERROR_M(exception, _), BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, exception), BlockWorker; -handle_query_result(_Id, _, ?RESOURCE_ERROR_M(NotWorking, _)) when +handle_query_result(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _) when NotWorking == not_connected; NotWorking == blocked -> true; -handle_query_result(_Id, BlockWorker, ?RESOURCE_ERROR_M(_, _)) -> +handle_query_result(_Id, ?RESOURCE_ERROR_M(_, _), BlockWorker) -> BlockWorker; -handle_query_result(Id, BlockWorker, {error, _}) -> +handle_query_result(Id, {error, _}, BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, failed), BlockWorker; -handle_query_result(Id, _BlockWorker, {resource_down, _}) -> +handle_query_result(Id, {resource_down, _}, _BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down), true. call_query(Id, Request) -> - do_call_query(on_query, Id, Request). + do_call_query(on_query, Id, Request, 1). call_batch_query(Id, Batch) -> - do_call_query(on_batch_query, Id, Batch). + do_call_query(on_batch_query, Id, Batch, length(Batch)). -do_call_query(Fun, Id, Data) -> +do_call_query(Fun, Id, Data, Count) -> case emqx_resource_manager:ets_lookup(Id) of {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> - ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Data)), + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, Count), try Mod:Fun(Id, Data, ResourceState) of %% if the callback module (connector) wants to return an error that %% makes the current resource goes into the `error` state, it should diff --git a/apps/emqx_resource/src/emqx_resource_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_worker_sup.erl new file mode 100644 index 000000000..a2b3a1ba5 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_worker_sup.erl @@ -0,0 +1,136 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_resource_worker_sup). +-behaviour(supervisor). + +%%%============================================================================= +%%% Exports and Definitions +%%%============================================================================= + +%% External API +-export([start_link/0]). + +-export([start_workers/2, stop_workers/2]). + +%% Callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +%%%============================================================================= +%%% API +%%%============================================================================= + +-spec start_link() -> supervisor:startlink_ret(). +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%%%============================================================================= +%%% Callbacks +%%%============================================================================= + +-spec init(list()) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}} | ignore. +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 100, + period => 30 + }, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +start_workers(ResId, Opts) -> + PoolSize = pool_size(Opts), + _ = ensure_worker_pool(ResId, hash, [{size, PoolSize}]), + lists:foreach( + fun(Idx) -> + _ = ensure_worker_added(ResId, Idx), + ok = ensure_worker_started(ResId, Idx, Opts) + end, + lists:seq(1, PoolSize) + ). + +stop_workers(ResId, Opts) -> + PoolSize = pool_size(Opts), + lists:foreach( + fun(Idx) -> + ensure_worker_removed(ResId, Idx) + end, + lists:seq(1, PoolSize) + ), + ensure_worker_pool_removed(ResId), + ok. + +%%%============================================================================= +%%% Internal +%%%============================================================================= +pool_size(Opts) -> + maps:get(worker_pool_size, Opts, erlang:system_info(schedulers_online)). + +ensure_worker_pool(ResId, Type, Opts) -> + try + gproc_pool:new(ResId, Type, Opts) + catch + error:exists -> ok + end, + ok. + +ensure_worker_added(ResId, Idx) -> + try + gproc_pool:add_worker(ResId, {ResId, Idx}, Idx) + catch + error:exists -> ok + end, + ok. + +-define(CHILD_ID(MOD, RESID, INDEX), {MOD, RESID, INDEX}). +ensure_worker_started(ResId, Idx, Opts) -> + Mod = emqx_resource_worker, + Spec = #{ + id => ?CHILD_ID(Mod, ResId, Idx), + start => {Mod, start_link, [ResId, Idx, Opts]}, + restart => transient, + shutdown => 5000, + type => worker, + modules => [Mod] + }, + case supervisor:start_child(emqx_resource_sup, Spec) of + {ok, _Pid} -> ok; + {error, {already_started, _}} -> ok; + {error, already_present} -> ok; + {error, _} = Err -> Err + end. + +ensure_worker_removed(ResId, Idx) -> + ChildId = ?CHILD_ID(emqx_resource_worker, ResId, Idx), + case supervisor:terminate_child(emqx_resource_sup, ChildId) of + ok -> + Res = supervisor:delete_child(emqx_resource_sup, ChildId), + _ = gproc_pool:remove_worker(ResId, {ResId, Idx}), + Res; + {error, not_found} -> + ok; + {error, Reason} -> + {error, Reason} + end. + +ensure_worker_pool_removed(ResId) -> + try + gproc_pool:delete(ResId) + catch + error:badarg -> ok + end, + ok. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 51e6bac43..915c59611 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -26,6 +26,7 @@ -define(TEST_RESOURCE, emqx_test_resource). -define(ID, <<"id">>). -define(DEFAULT_RESOURCE_GROUP, <<"default">>). +-define(RESOURCE_ERROR(REASON), {error, {resource_error, #{reason := REASON}}}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -80,7 +81,7 @@ t_create_remove(_) -> #{name => test_resource}, #{} ), - #{pid := Pid} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid)), @@ -110,7 +111,7 @@ t_create_remove_local(_) -> #{name => test_resource}, #{} ), - #{pid := Pid} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid)), @@ -127,7 +128,7 @@ t_create_remove_local(_) -> {error, _} = emqx_resource:remove_local(?ID), ?assertMatch( - {error, {emqx_resource, #{reason := not_found}}}, + ?RESOURCE_ERROR(not_created), emqx_resource:query(?ID, get_state) ), ?assertNot(is_process_alive(Pid)). @@ -143,23 +144,23 @@ t_do_not_start_after_created(_) -> %% the resource should remain `disconnected` after created timer:sleep(200), ?assertMatch( - {error, {emqx_resource, #{reason := not_connected}}}, + ?RESOURCE_ERROR(stopped), emqx_resource:query(?ID, get_state) ), ?assertMatch( - {ok, _, #{status := disconnected}}, + {ok, _, #{status := stopped}}, emqx_resource:get_instance(?ID) ), %% start the resource manually.. ok = emqx_resource:start(?ID), - #{pid := Pid} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid)), %% restart the resource ok = emqx_resource:restart(?ID), ?assertNot(is_process_alive(Pid)), - #{pid := Pid2} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid2}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid2)), ok = emqx_resource:remove_local(?ID), @@ -174,23 +175,10 @@ t_query(_) -> #{name => test_resource} ), - Pid = self(), - Success = fun() -> Pid ! success end, - Failure = fun() -> Pid ! failure end, - - #{pid := _} = emqx_resource:query(?ID, get_state), - #{pid := _} = emqx_resource:query(?ID, get_state, {[{Success, []}], [{Failure, []}]}), - #{pid := _} = emqx_resource:query(?ID, get_state, undefined), - #{pid := _} = emqx_resource:query(?ID, get_state_failed, undefined), - - receive - Message -> ?assertEqual(success, Message) - after 100 -> - ?assert(false) - end, + {ok, #{pid := _}} = emqx_resource:query(?ID, get_state), ?assertMatch( - {error, {emqx_resource, #{reason := not_found}}}, + ?RESOURCE_ERROR(not_created), emqx_resource:query(<<"unknown">>, get_state) ), @@ -201,11 +189,14 @@ t_healthy_timeout(_) -> ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, - #{name => <<"test_resource">>}, - #{health_check_timeout => 200} + #{name => <<"bad_not_atom_name">>, register => true}, + %% the ?TEST_RESOURCE always returns the `Mod:on_get_status/2` 300ms later. + #{health_check_interval => 200} + ), + ?assertMatch( + ?RESOURCE_ERROR(not_connected), + emqx_resource:query(?ID, get_state) ), - timer:sleep(500), - ok = emqx_resource:remove_local(?ID). t_healthy(_) -> @@ -213,11 +204,9 @@ t_healthy(_) -> ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, - #{name => <<"test_resource">>} + #{name => test_resource} ), - timer:sleep(400), - - #{pid := Pid} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), timer:sleep(300), emqx_resource:set_resource_status_connecting(?ID), @@ -229,10 +218,10 @@ t_healthy(_) -> erlang:exit(Pid, shutdown), - ?assertEqual({ok, connecting}, emqx_resource:health_check(?ID)), + ?assertEqual({ok, disconnected}, emqx_resource:health_check(?ID)), ?assertMatch( - [#{status := connecting}], + [#{status := disconnected}], emqx_resource:list_instances_verbose() ), @@ -260,7 +249,7 @@ t_stop_start(_) -> #{} ), - #{pid := Pid0} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid0)), @@ -269,14 +258,14 @@ t_stop_start(_) -> ?assertNot(is_process_alive(Pid0)), ?assertMatch( - {error, {emqx_resource, #{reason := not_connected}}}, + ?RESOURCE_ERROR(stopped), emqx_resource:query(?ID, get_state) ), ok = emqx_resource:restart(?ID), timer:sleep(300), - #{pid := Pid1} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid1)). @@ -302,7 +291,7 @@ t_stop_start_local(_) -> #{} ), - #{pid := Pid0} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid0)), @@ -311,13 +300,13 @@ t_stop_start_local(_) -> ?assertNot(is_process_alive(Pid0)), ?assertMatch( - {error, {emqx_resource, #{reason := not_connected}}}, + ?RESOURCE_ERROR(stopped), emqx_resource:query(?ID, get_state) ), ok = emqx_resource:restart(?ID), - #{pid := Pid1} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state), ?assert(is_process_alive(Pid1)). @@ -368,17 +357,17 @@ create_dry_run_local_succ() -> ?assertEqual(undefined, whereis(test_resource)). t_create_dry_run_local_failed(_) -> - {Res1, _} = emqx_resource:create_dry_run_local( + Res1 = emqx_resource:create_dry_run_local( ?TEST_RESOURCE, - #{cteate_error => true} + #{create_error => true} ), - ?assertEqual(error, Res1), + ?assertMatch({error, _}, Res1), - {Res2, _} = emqx_resource:create_dry_run_local( + Res2 = emqx_resource:create_dry_run_local( ?TEST_RESOURCE, #{name => test_resource, health_check_error => true} ), - ?assertEqual(error, Res2), + ?assertMatch({error, _}, Res2), Res3 = emqx_resource:create_dry_run_local( ?TEST_RESOURCE, @@ -400,7 +389,7 @@ t_reset_metrics(_) -> #{name => test_resource} ), - #{pid := Pid} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), emqx_resource:reset_metrics(?ID), ?assert(is_process_alive(Pid)), ok = emqx_resource:remove(?ID), diff --git a/apps/emqx_resource/test/emqx_test_resource.erl b/apps/emqx_resource/test/emqx_test_resource.erl index c23f87d50..569579d27 100644 --- a/apps/emqx_resource/test/emqx_test_resource.erl +++ b/apps/emqx_resource/test/emqx_test_resource.erl @@ -24,7 +24,7 @@ -export([ on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -50,24 +50,20 @@ on_start(_InstId, #{create_error := true}) -> error("some error"); on_start(InstId, #{name := Name, stop_error := true} = Opts) -> Register = maps:get(register, Opts, false), - {ok, #{ - name => Name, + {ok, Opts#{ id => InstId, stop_error => true, pid => spawn_dummy_process(Name, Register) }}; -on_start(InstId, #{name := Name, health_check_error := true} = Opts) -> +on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), - {ok, #{ - name => Name, + {ok, Opts#{ id => InstId, - health_check_error => true, pid => spawn_dummy_process(Name, Register) }}; on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), - {ok, #{ - name => Name, + {ok, Opts#{ id => InstId, pid => spawn_dummy_process(Name, Register) }}. @@ -78,12 +74,10 @@ on_stop(_InstId, #{pid := Pid}) -> erlang:exit(Pid, shutdown), ok. -on_query(_InstId, get_state, AfterQuery, State) -> - emqx_resource:query_success(AfterQuery), - State; -on_query(_InstId, get_state_failed, AfterQuery, State) -> - emqx_resource:query_failed(AfterQuery), - State. +on_query(_InstId, get_state, State) -> + {ok, State}; +on_query(_InstId, get_state_failed, State) -> + {error, State}. on_get_status(_InstId, #{health_check_error := true}) -> disconnected; @@ -91,10 +85,11 @@ on_get_status(_InstId, #{pid := Pid}) -> timer:sleep(300), case is_process_alive(Pid) of true -> connected; - false -> connecting + false -> disconnected end. spawn_dummy_process(Name, Register) -> + ct:pal("---- Register Name: ~p", [Name]), spawn( fun() -> true = From 0377d3cf61469aa51cbe367bdfb5dd1a2768e518 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 29 Jul 2022 18:49:57 +0800 Subject: [PATCH 32/71] fix: update existing testcases for new emqx_resource --- .../src/simple_authn/emqx_authn_mongodb.erl | 2 +- apps/emqx_authz/src/emqx_authz_mongodb.erl | 4 +- .../test/emqx_bridge_api_SUITE.erl | 116 ++++++++++-------- .../src/emqx_connector_mqtt.erl | 5 +- .../test/emqx_connector_mongo_SUITE.erl | 8 +- apps/emqx_resource/src/emqx_resource.app.src | 2 +- apps/emqx_resource/src/emqx_resource.erl | 5 +- .../src/emqx_resource_worker.erl | 26 ++-- 8 files changed, 98 insertions(+), 70 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index ff9c97717..1351ae0dd 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -164,7 +164,7 @@ authenticate( ) -> Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential), case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of - undefined -> + {ok, undefined} -> ignore; {error, Reason} -> ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{ diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index ac450e4cc..a1e1b8136 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -92,9 +92,9 @@ authorize( resource_id => ResourceID }), nomatch; - [] -> + {ok, []} -> nomatch; - Rows -> + {ok, Rows} -> Rules = [ emqx_authz_rule:compile({Permission, all, Action, Topics}) || #{ diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index c048a13fe..9346fb9c0 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("common_test/include/ct.hrl"). -define(CONF_DEFAULT, <<"bridges: {}">>). -define(BRIDGE_TYPE, <<"webhook">>). --define(BRIDGE_NAME, <<"test_bridge">>). +-define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))). -define(URL(PORT, PATH), list_to_binary( io_lib:format( @@ -78,8 +78,12 @@ set_special_configs(_) -> init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - Config. -end_per_testcase(_, _Config) -> + {Port, Sock, Acceptor} = start_http_server(fun handle_fun_200_ok/2), + [{port, Port}, {sock, Sock}, {acceptor, Acceptor} | Config]. +end_per_testcase(_, Config) -> + Sock = ?config(sock, Config), + Acceptor = ?config(acceptor, Config), + stop_http_server(Sock, Acceptor), clear_resources(), ok. @@ -95,31 +99,39 @@ clear_resources() -> %% HTTP server for testing %%------------------------------------------------------------------------------ start_http_server(HandleFun) -> + process_flag(trap_exit, true), Parent = self(), - spawn_link(fun() -> - {Port, Sock} = listen_on_random_port(), - Parent ! {port, Port}, - loop(Sock, HandleFun, Parent) + {Port, Sock} = listen_on_random_port(), + Acceptor = spawn_link(fun() -> + accept_loop(Sock, HandleFun, Parent) end), - receive - {port, Port} -> Port - after 2000 -> error({timeout, start_http_server}) - end. + timer:sleep(100), + {Port, Sock, Acceptor}. + +stop_http_server(Sock, Acceptor) -> + exit(Acceptor, kill), + gen_tcp:close(Sock). listen_on_random_port() -> Min = 1024, Max = 65000, + rand:seed(exsplus, erlang:timestamp()), Port = rand:uniform(Max - Min) + Min, - case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of + case + gen_tcp:listen(Port, [ + binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000} + ]) + of {ok, Sock} -> {Port, Sock}; {error, eaddrinuse} -> listen_on_random_port() end. -loop(Sock, HandleFun, Parent) -> +accept_loop(Sock, HandleFun, Parent) -> + process_flag(trap_exit, true), {ok, Conn} = gen_tcp:accept(Sock), - Handler = spawn(fun() -> HandleFun(Conn, Parent) end), + Handler = spawn_link(fun() -> HandleFun(Conn, Parent) end), gen_tcp:controlling_process(Conn, Handler), - loop(Sock, HandleFun, Parent). + accept_loop(Sock, HandleFun, Parent). make_response(CodeStr, Str) -> B = iolist_to_binary(Str), @@ -138,7 +150,9 @@ handle_fun_200_ok(Conn, Parent) -> Parent ! {http_server, received, Req}, gen_tcp:send(Conn, make_response("200 OK", "Request OK")), handle_fun_200_ok(Conn, Parent); - {error, closed} -> + {error, Reason} -> + ct:pal("the http handler recv error: ~p", [Reason]), + timer:sleep(100), gen_tcp:close(Conn) end. @@ -153,24 +167,25 @@ parse_http_request(ReqStr0) -> %% Testcases %%------------------------------------------------------------------------------ -t_http_crud_apis(_) -> - Port = start_http_server(fun handle_fun_200_ok/2), +t_http_crud_apis(Config) -> + Port = ?config(port, Config), %% assert we there's no bridges at first {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), %% then we add a webhook bridge, using POST %% POST /bridges/ will create a bridge URL1 = ?URL(Port, "path1"), + Name = ?BRIDGE_NAME, {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) ), %ct:pal("---bridge: ~p", [Bridge]), #{ <<"type">> := ?BRIDGE_TYPE, - <<"name">> := ?BRIDGE_NAME, + <<"name">> := Name, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _], @@ -179,7 +194,7 @@ t_http_crud_apis(_) -> <<"url">> := URL1 } = jsx:decode(Bridge), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), %% send an message to emqx and the message should be forwarded to the HTTP server Body = <<"my msg">>, emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)), @@ -203,12 +218,12 @@ t_http_crud_apis(_) -> {ok, 200, Bridge2} = request( put, uri(["bridges", BridgeID]), - ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name) ), ?assertMatch( #{ <<"type">> := ?BRIDGE_TYPE, - <<"name">> := ?BRIDGE_NAME, + <<"name">> := Name, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _], @@ -225,7 +240,7 @@ t_http_crud_apis(_) -> [ #{ <<"type">> := ?BRIDGE_TYPE, - <<"name">> := ?BRIDGE_NAME, + <<"name">> := Name, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _], @@ -242,7 +257,7 @@ t_http_crud_apis(_) -> ?assertMatch( #{ <<"type">> := ?BRIDGE_TYPE, - <<"name">> := ?BRIDGE_NAME, + <<"name">> := Name, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _], @@ -275,7 +290,7 @@ t_http_crud_apis(_) -> {ok, 404, ErrMsg2} = request( put, uri(["bridges", BridgeID]), - ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name) ), ?assertMatch( #{ @@ -286,29 +301,28 @@ t_http_crud_apis(_) -> ), ok. -t_start_stop_bridges(_) -> - lists:foreach( - fun(Type) -> - do_start_stop_bridges(Type) - end, - [node, cluster] - ). +t_start_stop_bridges_node(Config) -> + do_start_stop_bridges(node, Config). -do_start_stop_bridges(Type) -> +t_start_stop_bridges_cluster(Config) -> + do_start_stop_bridges(cluster, Config). + +do_start_stop_bridges(Type, Config) -> %% assert we there's no bridges at first {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - Port = start_http_server(fun handle_fun_200_ok/2), + Port = ?config(port, Config), URL1 = ?URL(Port, "abc"), + Name = atom_to_binary(Type), {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) ), %ct:pal("the bridge ==== ~p", [Bridge]), #{ <<"type">> := ?BRIDGE_TYPE, - <<"name">> := ?BRIDGE_NAME, + <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], @@ -316,11 +330,11 @@ do_start_stop_bridges(Type) -> <<"node_metrics">> := [_ | _], <<"url">> := URL1 } = jsx:decode(Bridge), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), %% stop it {ok, 200, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>), {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"disconnected">>}, jsx:decode(Bridge2)), + ?assertMatch(#{<<"status">> := <<"stopped">>}, jsx:decode(Bridge2)), %% start again {ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), @@ -339,21 +353,22 @@ do_start_stop_bridges(Type) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). -t_enable_disable_bridges(_) -> +t_enable_disable_bridges(Config) -> %% assert we there's no bridges at first {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - Port = start_http_server(fun handle_fun_200_ok/2), + Name = ?BRIDGE_NAME, + Port = ?config(port, Config), URL1 = ?URL(Port, "abc"), {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) ), %ct:pal("the bridge ==== ~p", [Bridge]), #{ <<"type">> := ?BRIDGE_TYPE, - <<"name">> := ?BRIDGE_NAME, + <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], @@ -361,11 +376,11 @@ t_enable_disable_bridges(_) -> <<"node_metrics">> := [_ | _], <<"url">> := URL1 } = jsx:decode(Bridge), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), %% disable it {ok, 200, <<>>} = request(post, operation_path(cluster, disable, BridgeID), <<"">>), {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"disconnected">>}, jsx:decode(Bridge2)), + ?assertMatch(#{<<"status">> := <<"stopped">>}, jsx:decode(Bridge2)), %% enable again {ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), @@ -391,21 +406,22 @@ t_enable_disable_bridges(_) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). -t_reset_bridges(_) -> +t_reset_bridges(Config) -> %% assert we there's no bridges at first {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - Port = start_http_server(fun handle_fun_200_ok/2), + Name = ?BRIDGE_NAME, + Port = ?config(port, Config), URL1 = ?URL(Port, "abc"), {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) ), %ct:pal("the bridge ==== ~p", [Bridge]), #{ <<"type">> := ?BRIDGE_TYPE, - <<"name">> := ?BRIDGE_NAME, + <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], @@ -413,7 +429,7 @@ t_reset_bridges(_) -> <<"node_metrics">> := [_ | _], <<"url">> := URL1 } = jsx:decode(Bridge), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), {ok, 200, <<"Reset success">>} = request(put, uri(["bridges", BridgeID, "reset_metrics"]), []), %% delete the bridge diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 98635de3f..0957e3c18 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -134,7 +134,8 @@ drop_bridge(Name) -> %% When use this bridge as a data source, ?MODULE:on_message_received will be called %% if the bridge received msgs from the remote broker. on_message_received(Msg, HookPoint, InstId) -> - _ = emqx_resource:query(InstId, {message_received, Msg}), + emqx_resource:inc_matched(InstId), + emqx_resource:inc_success(InstId), emqx:run_hook(HookPoint, [Msg]). %% =================================================================== @@ -181,8 +182,6 @@ on_stop(_InstId, #{name := InstanceId}) -> }) end. -on_query(_InstId, {message_received, _Msg}, _State) -> - ok; on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) -> ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg), diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl index d76b8420a..2ad1f5f8e 100644 --- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl @@ -85,8 +85,8 @@ perform_lifecycle_check(PoolName, InitialConfig) -> emqx_resource:get_instance(PoolName), ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), % % Perform query as further check that the resource is working as expected - ?assertMatch([], emqx_resource:query(PoolName, test_query_find())), - ?assertMatch(undefined, emqx_resource:query(PoolName, test_query_find_one())), + ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())), + ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())), ?assertEqual(ok, emqx_resource:stop(PoolName)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. @@ -108,8 +108,8 @@ perform_lifecycle_check(PoolName, InitialConfig) -> {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = emqx_resource:get_instance(PoolName), ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch([], emqx_resource:query(PoolName, test_query_find())), - ?assertMatch(undefined, emqx_resource:query(PoolName, test_query_find_one())), + ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())), + ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())), % Stop and remove the resource in one go. ?assertEqual(ok, emqx_resource:remove_local(PoolName)), ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 1bfd02323..b688e3c11 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 081264315..af047060c 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -103,7 +103,7 @@ list_group_instances/1 ]). --export([inc_metrics_funcs/1, inc_success/1, inc_failed/1]). +-export([inc_metrics_funcs/1, inc_matched/1, inc_success/1, inc_failed/1]). -optional_callbacks([ on_query/3, @@ -393,6 +393,9 @@ check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> %% ================================================================================= +inc_matched(ResId) -> + emqx_metrics_worker:inc(?RES_METRICS, ResId, matched). + inc_success(ResId) -> emqx_metrics_worker:inc(?RES_METRICS, ResId, success). diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index ae0d24313..9ab7fb749 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -44,7 +44,7 @@ -export([running/3, blocked/3]). --export([queue_item_marshaller/1]). +-export([queue_item_marshaller/1, estimate_size/1]). -define(RESUME_INTERVAL, 15000). @@ -112,6 +112,7 @@ init({Id, Index, Opts}) -> replayq:open(#{ dir => disk_queue_dir(Id, Index), seg_bytes => 10000000, + sizer => fun ?MODULE:estimate_size/1, marshaller => fun ?MODULE:queue_item_marshaller/1 }); false -> @@ -172,6 +173,9 @@ queue_item_marshaller(?Q_ITEM(_) = I) -> queue_item_marshaller(Bin) when is_binary(Bin) -> binary_to_term(Bin). +estimate_size(QItem) -> + size(queue_item_marshaller(QItem)). + %%============================================================================== pick_query(Fun, Id, Key, Query) -> try gproc_pool:pick_worker(Id, Key) of @@ -277,12 +281,6 @@ reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) -> gen_statem:reply(From, Result), handle_query_result(Id, Result, BlockWorker). -handle_query_result(Id, ok, BlockWorker) -> - emqx_metrics_worker:inc(?RES_METRICS, Id, success), - BlockWorker; -handle_query_result(Id, {ok, _}, BlockWorker) -> - emqx_metrics_worker:inc(?RES_METRICS, Id, success), - BlockWorker; handle_query_result(Id, ?RESOURCE_ERROR_M(exception, _), BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, exception), BlockWorker; @@ -297,7 +295,12 @@ handle_query_result(Id, {error, _}, BlockWorker) -> BlockWorker; handle_query_result(Id, {resource_down, _}, _BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down), - true. + true; +handle_query_result(Id, Result, BlockWorker) -> + %% assert + true = is_ok_result(Result), + emqx_metrics_worker:inc(?RES_METRICS, Id, success), + BlockWorker. call_query(Id, Request) -> do_call_query(on_query, Id, Request, 1). @@ -339,6 +342,13 @@ maybe_expand_batch_result(Result, Batch) -> %%============================================================================== +is_ok_result(ok) -> + true; +is_ok_result(R) when is_tuple(R) -> + erlang:element(1, R) == ok; +is_ok_result(_) -> + false. + -spec name(id(), integer()) -> atom(). name(Id, Index) -> Mod = atom_to_list(?MODULE), From d3950b9534ae9e220ad7e6a550f3f549ca4baf19 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 29 Jul 2022 23:26:54 +0800 Subject: [PATCH 33/71] fix(resource): make option 'queue_enabled' disabled by default --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_authz/src/emqx_authz.app.src | 2 +- .../emqx_connector/src/emqx_connector.app.src | 2 +- .../src/emqx_resource_worker.erl | 27 +++++++------------ .../test/emqx_resource_SUITE.erl | 4 +++ 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 8087e822f..ef67b9a14 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index ed19b15a8..e40b5e64c 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 007962da3..cce266966 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "An OTP application"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 9ab7fb749..055fbfc53 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -73,7 +73,7 @@ -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) -> {{from(), result()}, NewCbState :: term()}. -callback_mode() -> [state_functions, state_enter]. +callback_mode() -> [state_functions]. start_link(Id, Index, Opts) -> gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []). @@ -107,7 +107,7 @@ init({Id, Index, Opts}) -> true = gproc_pool:connect_worker(Id, {Id, Index}), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), Queue = - case maps:get(queue_enabled, Opts, true) of + case maps:get(queue_enabled, Opts, false) of true -> replayq:open(#{ dir => disk_queue_dir(Id, Index), @@ -131,8 +131,6 @@ init({Id, Index, Opts}) -> }, {ok, blocked, St, {next_event, cast, resume}}. -running(enter, _, _St) -> - keep_state_and_data; running(cast, resume, _St) -> keep_state_and_data; running(cast, block, St) -> @@ -149,8 +147,6 @@ running(info, Info, _St) -> ?SLOG(error, #{msg => unexpected_msg, info => Info}), keep_state_and_data. -blocked(enter, _, _St) -> - keep_state_and_data; blocked(cast, block, _St) -> keep_state_and_data; blocked(cast, resume, St) -> @@ -218,28 +214,25 @@ drop_head(Q) -> ok = replayq:ack(Q1, AckRef), Q1. -query_or_acc(From, Request, #{batch_enabled := true} = St) -> - acc_query(From, Request, St); -query_or_acc(From, Request, #{batch_enabled := false} = St) -> - send_query(From, Request, St). - -acc_query(From, Request, #{acc := Acc, acc_left := Left} = St0) -> +query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Left} = St0) -> Acc1 = [?QUERY(From, Request) | Acc], St = St0#{acc := Acc1, acc_left := Left - 1}, case Left =< 1 of true -> flush(St); false -> {keep_state, ensure_flush_timer(St)} - end. - -send_query(From, Request, #{id := Id, queue := Q} = St) -> - Result = call_query(Id, Request), - case reply_caller(Id, ?REPLY(From, Request, Result), false) of + end; +query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id} = St) -> + case send_query(From, Request, Id) of true -> {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}}; false -> {keep_state, St} end. +send_query(From, Request, Id) -> + Result = call_query(Id, Request), + reply_caller(Id, ?REPLY(From, Request, Result), false). + flush(#{acc := []} = St) -> {keep_state, St}; flush( diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 915c59611..278f556ef 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -347,6 +347,10 @@ t_create_dry_run_local(_) -> [] = ets:match(emqx_resource_manager, {{owner, '$1'}, '_'}). create_dry_run_local_succ() -> + case whereis(test_resource) of + undefined -> ok; + Pid -> exit(Pid, kill) + end, ?assertEqual( ok, emqx_resource:create_dry_run_local( From 75adba07813de06bbf50196943bc5919e6f296c9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 30 Jul 2022 16:35:12 +0800 Subject: [PATCH 34/71] fix: increase resource metrics using the resource id --- .../emqx_connector/src/emqx_connector_mqtt.erl | 9 +++++---- .../src/emqx_resource_manager.erl | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 0957e3c18..66682caeb 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -133,9 +133,9 @@ drop_bridge(Name) -> %% =================================================================== %% When use this bridge as a data source, ?MODULE:on_message_received will be called %% if the bridge received msgs from the remote broker. -on_message_received(Msg, HookPoint, InstId) -> - emqx_resource:inc_matched(InstId), - emqx_resource:inc_success(InstId), +on_message_received(Msg, HookPoint, ResId) -> + emqx_resource:inc_matched(ResId), + emqx_resource:inc_success(ResId), emqx:run_hook(HookPoint, [Msg]). %% =================================================================== @@ -206,11 +206,12 @@ make_sub_confs(EmptyMap, _) when map_size(EmptyMap) == 0 -> make_sub_confs(undefined, _) -> undefined; make_sub_confs(SubRemoteConf, InstId) -> + ResId = emqx_resource_manager:manager_id_to_resource_id(InstId), case maps:take(hookpoint, SubRemoteConf) of error -> SubRemoteConf; {HookPoint, SubConf} -> - MFA = {?MODULE, on_message_received, [HookPoint, InstId]}, + MFA = {?MODULE, on_message_received, [HookPoint, ResId]}, SubConf#{on_message_received => MFA} end. diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index b5bcbd330..7dad85d5b 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -38,8 +38,12 @@ list_group/1, ets_lookup/1, get_metrics/1, - reset_metrics/1, - set_resource_status_connecting/1 + reset_metrics/1 +]). + +-export([ + set_resource_status_connecting/1, + manager_id_to_resource_id/1 ]). % Server @@ -64,6 +68,13 @@ %% API %%------------------------------------------------------------------------------ +make_manager_id(ResId) -> + emqx_resource:generate_id(ResId). + +manager_id_to_resource_id(MgrId) -> + [ResId, _Index] = string:split(MgrId, ":", trailing), + ResId. + %% @doc Called from emqx_resource when starting a resource instance. %% %% Triggers the emqx_resource_manager_sup supervisor to actually create @@ -455,9 +466,6 @@ stop_resource(Data) -> _ = maybe_clear_alarm(Data#data.id), ok. -make_manager_id(ResId) -> - emqx_resource:generate_id(ResId). - make_test_id() -> RandId = iolist_to_binary(emqx_misc:gen_id(16)), <>. From 8f0954837bdca5037bac40f799c531c8fe3e7a9e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 30 Jul 2022 22:42:26 +0800 Subject: [PATCH 35/71] fix: update test cases for emqx_connector --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl | 2 +- apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl | 2 +- apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl | 2 +- apps/emqx_connector/test/emqx_connector_redis_SUITE.erl | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index fed9e6bc2..b7e65a042 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl index 2ad1f5f8e..e918be84a 100644 --- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl @@ -95,7 +95,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> status := StoppedStatus }} = emqx_resource:get_instance(PoolName), - ?assertEqual(StoppedStatus, disconnected), + ?assertEqual(stopped, StoppedStatus), ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), diff --git a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl index d7f5cec63..3fd7191b9 100644 --- a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl @@ -101,7 +101,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> status := StoppedStatus }} = emqx_resource:get_instance(PoolName), - ?assertEqual(StoppedStatus, disconnected), + ?assertEqual(stopped, StoppedStatus), ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), diff --git a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl index d99d8ab6c..9442a1810 100644 --- a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl @@ -95,7 +95,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> status := StoppedStatus }} = emqx_resource:get_instance(PoolName), - ?assertEqual(StoppedStatus, disconnected), + ?assertEqual(stopped, StoppedStatus), ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index 4770bbeee..a60702036 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -117,7 +117,7 @@ perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) -> status := StoppedStatus }} = emqx_resource:get_instance(PoolName), - ?assertEqual(StoppedStatus, disconnected), + ?assertEqual(stopped, StoppedStatus), ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), From a2afdeeb48c3f7ee5c1313017fde4b6caf85281d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 1 Aug 2022 13:06:28 +0800 Subject: [PATCH 36/71] feat: add test cases for batching query --- apps/emqx_resource/README.md | 2 +- ...t_resource.erl => emqx_connector_demo.erl} | 85 ++++++++++++++----- .../test/emqx_resource_SUITE.erl | 47 +++++++++- 3 files changed, 113 insertions(+), 21 deletions(-) rename apps/emqx_resource/test/{emqx_test_resource.erl => emqx_connector_demo.erl} (54%) diff --git a/apps/emqx_resource/README.md b/apps/emqx_resource/README.md index 04f3c2205..0f61df7ff 100644 --- a/apps/emqx_resource/README.md +++ b/apps/emqx_resource/README.md @@ -14,5 +14,5 @@ the config operations (like config validation, config dump back to files), and t And we put all the `specific` codes to the callback modules. See -* `test/emqx_test_resource.erl` for a minimal `emqx_resource` implementation; +* `test/emqx_connector_demo.erl` for a minimal `emqx_resource` implementation; * `test/emqx_resource_SUITE.erl` for examples of `emqx_resource` usage. diff --git a/apps/emqx_resource/test/emqx_test_resource.erl b/apps/emqx_resource/test/emqx_connector_demo.erl similarity index 54% rename from apps/emqx_resource/test/emqx_test_resource.erl rename to apps/emqx_resource/test/emqx_connector_demo.erl index 569579d27..e9c77e915 100644 --- a/apps/emqx_resource/test/emqx_test_resource.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_test_resource). +-module(emqx_connector_demo). -include_lib("typerefl/include/types.hrl"). @@ -25,9 +25,12 @@ on_start/2, on_stop/2, on_query/3, + on_batch_query/3, on_get_status/2 ]). +-export([counter_loop/1]). + %% callbacks for emqx_resource config schema -export([roots/0]). @@ -53,19 +56,19 @@ on_start(InstId, #{name := Name, stop_error := true} = Opts) -> {ok, Opts#{ id => InstId, stop_error => true, - pid => spawn_dummy_process(Name, Register) + pid => spawn_counter_process(Name, Register) }}; on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), {ok, Opts#{ id => InstId, - pid => spawn_dummy_process(Name, Register) + pid => spawn_counter_process(Name, Register) }}; on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), {ok, Opts#{ id => InstId, - pid => spawn_dummy_process(Name, Register) + pid => spawn_counter_process(Name, Register) }}. on_stop(_InstId, #{stop_error := true}) -> @@ -77,7 +80,44 @@ on_stop(_InstId, #{pid := Pid}) -> on_query(_InstId, get_state, State) -> {ok, State}; on_query(_InstId, get_state_failed, State) -> - {error, State}. + {error, State}; +on_query(_InstId, {inc_counter, N}, #{pid := Pid}) -> + Pid ! {inc, N}, + ok; +on_query(_InstId, get_counter, #{pid := Pid}) -> + ReqRef = make_ref(), + From = {self(), ReqRef}, + Pid ! {From, get}, + receive + {ReqRef, Num} -> {ok, Num} + after 1000 -> + {error, timeout} + end. + +on_batch_query(InstId, BatchReq, State) -> + %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed. + case hd(BatchReq) of + {_From, {inc_counter, _}} -> + batch_inc_counter(InstId, BatchReq, State); + {_From, get_counter} -> + batch_get_counter(InstId, State) + end. + +batch_inc_counter(InstId, BatchReq, State) -> + TotalN = lists:foldl( + fun + ({_From, {inc_counter, N}}, Total) -> + Total + N; + ({_From, Req}, _Total) -> + error({mixed_requests_not_allowed, {inc_counter, Req}}) + end, + 0, + BatchReq + ), + on_query(InstId, {inc_counter, TotalN}, State). + +batch_get_counter(InstId, State) -> + on_query(InstId, get_counter, State). on_get_status(_InstId, #{health_check_error := true}) -> disconnected; @@ -88,18 +128,25 @@ on_get_status(_InstId, #{pid := Pid}) -> false -> disconnected end. -spawn_dummy_process(Name, Register) -> +spawn_counter_process(Name, Register) -> + Pid = spawn_link(?MODULE, counter_loop, [#{counter => 0}]), + true = maybe_register(Name, Pid, Register), + Pid. + +counter_loop(#{counter := Num} = State) -> + NewState = + receive + {inc, N} -> + #{counter => Num + N}; + {{FromPid, ReqRef}, get} -> + FromPid ! {ReqRef, Num}, + State + end, + counter_loop(NewState). + +maybe_register(Name, Pid, true) -> ct:pal("---- Register Name: ~p", [Name]), - spawn( - fun() -> - true = - case Register of - true -> register(Name, self()); - _ -> true - end, - Ref = make_ref(), - receive - Ref -> ok - end - end - ). + ct:pal("---- whereis(): ~p", [whereis(Name)]), + erlang:register(Name, Pid); +maybe_register(_Name, _Pid, false) -> + true. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 278f556ef..d05d4baf7 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -23,7 +23,7 @@ -include("emqx_resource.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). --define(TEST_RESOURCE, emqx_test_resource). +-define(TEST_RESOURCE, emqx_connector_demo). -define(ID, <<"id">>). -define(DEFAULT_RESOURCE_GROUP, <<"default">>). -define(RESOURCE_ERROR(REASON), {error, {resource_error, #{reason := REASON}}}). @@ -184,6 +184,51 @@ t_query(_) -> ok = emqx_resource:remove_local(?ID). +t_query_counter(_) -> + {ok, _} = emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, register => true} + ), + + {ok, 0} = emqx_resource:query(?ID, get_counter), + ok = emqx_resource:query(?ID, {inc_counter, 1}), + {ok, 1} = emqx_resource:query(?ID, get_counter), + ok = emqx_resource:query(?ID, {inc_counter, 5}), + {ok, 6} = emqx_resource:query(?ID, get_counter), + + ok = emqx_resource:remove_local(?ID). + +t_batch_query_counter(_) -> + {ok, _} = emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, register => true, batch_enabled => true} + ), + + {ok, 0} = emqx_resource:query(?ID, get_counter), + Parent = self(), + Pids = [ + erlang:spawn(fun() -> + ok = emqx_resource:query(?ID, {inc_counter, 1}), + Parent ! {complete, self()} + end) + || _ <- lists:seq(1, 1000) + ], + [ + receive + {complete, Pid} -> ok + after 1000 -> + ct:fail({wait_for_query_timeout, Pid}) + end + || Pid <- Pids + ], + {ok, 1000} = emqx_resource:query(?ID, get_counter), + + ok = emqx_resource:remove_local(?ID). + t_healthy_timeout(_) -> {ok, _} = emqx_resource:create_local( ?ID, From f1419d52f1390036b1894ef494ff1f3c9f1628f2 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 2 Aug 2022 18:39:18 +0800 Subject: [PATCH 37/71] fix(resource): remove resource at the end of each test --- apps/emqx_resource/test/emqx_resource_SUITE.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index d05d4baf7..4c5a36f4b 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -36,6 +36,8 @@ groups() -> init_per_testcase(_, Config) -> Config. +end_per_testcase(_, _Config) -> + _ = emqx_resource:remove(?ID). init_per_suite(Config) -> code:ensure_loaded(?TEST_RESOURCE), From 35fe70b887e0d0f3bd9c235fabff822596a11e40 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 8 Aug 2022 17:52:44 +0800 Subject: [PATCH 38/71] feat: support aysnc callback to connector modules --- .../emqx_authn_jwks_connector.erl | 3 + .../test/emqx_authz_api_cache_SUITE.erl | 2 + .../src/emqx_connector_http.erl | 3 + .../src/emqx_connector_ldap.erl | 3 + .../src/emqx_connector_mongo.erl | 3 + .../src/emqx_connector_mqtt.erl | 3 + .../src/emqx_connector_mysql.erl | 3 + .../src/emqx_connector_pgsql.erl | 3 + .../src/emqx_connector_redis.erl | 3 + apps/emqx_resource/include/emqx_resource.hrl | 1 + apps/emqx_resource/src/emqx_resource.erl | 11 +- .../src/emqx_resource_manager.erl | 5 +- .../src/emqx_resource_worker.erl | 210 +++++++++++------- .../test/emqx_connector_demo.erl | 9 +- 14 files changed, 173 insertions(+), 89 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index cd8451ac9..480950143 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -22,6 +22,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -31,6 +32,8 @@ -define(DEFAULT_POOL_SIZE, 8). +callback_mode() -> always_sync. + on_start(InstId, Opts) -> PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolOpts = [ diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 306fe3f13..0c49cc03a 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -23,6 +23,8 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +suite() -> [{timetrap, {seconds, 60}}]. + all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index e0d5ccfe0..c5a1b89db 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -164,6 +165,8 @@ ref(Field) -> hoconsc:ref(?MODULE, Field). %% =================================================================== +callback_mode() -> always_sync. + on_start( InstId, #{ diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 51d18b534..d53c0e41b 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -42,6 +43,8 @@ roots() -> fields(_) -> []. %% =================================================================== +callback_mode() -> always_sync. + on_start( InstId, #{ diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index db8b1e632..07208545f 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -139,6 +140,8 @@ mongo_fields() -> %% =================================================================== +callback_mode() -> always_sync. + on_start( InstId, Config = #{ diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 66682caeb..e37f6a9a2 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -24,6 +24,7 @@ %% API and callbacks for supervisor -export([ + callback_mode/0, start_link/0, init/1, create_bridge/1, @@ -139,6 +140,8 @@ on_message_received(Msg, HookPoint, ResId) -> emqx:run_hook(HookPoint, [Msg]). %% =================================================================== +callback_mode() -> always_sync. + on_start(InstId, Conf) -> InstanceId = binary_to_atom(InstId, utf8), ?SLOG(info, #{ diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index e818bc6ef..b379e511c 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -24,6 +24,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -73,6 +74,8 @@ server(desc) -> ?DESC("server"); server(_) -> undefined. %% =================================================================== +callback_mode() -> always_sync. + -spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. on_start( InstId, diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index d31c1316f..4b188e5a5 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -27,6 +27,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -66,6 +67,8 @@ server(desc) -> ?DESC("server"); server(_) -> undefined. %% =================================================================== +callback_mode() -> always_sync. + on_start( InstId, #{ diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 4826a170b..fae628d9e 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -112,6 +113,8 @@ servers(desc) -> ?DESC("servers"); servers(_) -> undefined. %% =================================================================== +callback_mode() -> always_sync. + on_start( InstId, #{ diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 13ffff587..c691789c2 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -25,6 +25,7 @@ -type resource_data() :: #{ id := resource_id(), mod := module(), + callback_mode := always_sync | async_if_possible, config := resource_config(), state := resource_state(), status := resource_status(), diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index af047060c..a54a77e19 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -75,8 +75,7 @@ %% stop the instance stop/1, %% query the instance - query/2, - query_async/3 + query/2 ]). %% Direct calls to the callback module @@ -224,12 +223,12 @@ reset_metrics(ResId) -> %% ================================================================================= -spec query(resource_id(), Request :: term()) -> Result :: term(). query(ResId, Request) -> - emqx_resource_worker:query(ResId, Request). + query(ResId, Request, #{}). --spec query_async(resource_id(), Request :: term(), emqx_resource_worker:reply_fun()) -> +-spec query(resource_id(), Request :: term(), emqx_resource_worker:query_opts()) -> Result :: term(). -query_async(ResId, Request, ReplyFun) -> - emqx_resource_worker:query_async(ResId, Request, ReplyFun). +query(ResId, Request, Opts) -> + emqx_resource_worker:query(ResId, Request, Opts). -spec start(resource_id()) -> ok | {error, Reason :: term()}. start(ResId) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 7dad85d5b..d6e5a1493 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -53,7 +53,7 @@ -export([init/1, callback_mode/0, handle_event/4, terminate/3]). % State record --record(data, {id, manager_id, group, mod, config, opts, status, state, error}). +-record(data, {id, manager_id, group, mod, callback_mode, config, opts, status, state, error}). -define(SHORT_HEALTHCHECK_INTERVAL, 1000). -define(HEALTHCHECK_INTERVAL, 15000). @@ -259,6 +259,7 @@ start_link(MgrId, ResId, Group, ResourceType, Config, Opts) -> manager_id = MgrId, group = Group, mod = ResourceType, + callback_mode = ResourceType:callback_mode(), config = Config, opts = Opts, status = connecting, @@ -559,10 +560,12 @@ maybe_reply(Actions, undefined, _Reply) -> maybe_reply(Actions, From, Reply) -> [{reply, From, Reply} | Actions]. +-spec data_record_to_external_map_with_metrics(#data{}) -> resource_data(). data_record_to_external_map_with_metrics(Data) -> #{ id => Data#data.id, mod => Data#data.mod, + callback_mode => Data#data.callback_mode, config => Data#data.config, status => Data#data.status, state => Data#data.state, diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 055fbfc53..2115fed86 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -27,11 +27,9 @@ -export([ start_link/3, - query/2, query/3, - query_async/3, - query_async/4, block/1, + block/2, resume/1 ]). @@ -46,6 +44,8 @@ -export([queue_item_marshaller/1, estimate_size/1]). +-export([reply_after_query/4, batch_reply_after_query/4]). + -define(RESUME_INTERVAL, 15000). %% count @@ -55,8 +55,8 @@ -define(Q_ITEM(REQUEST), {q_item, REQUEST}). --define(QUERY(FROM, REQUEST), {FROM, REQUEST}). --define(REPLY(FROM, REQUEST, RESULT), {FROM, REQUEST, RESULT}). +-define(QUERY(FROM, REQUEST), {query, FROM, REQUEST}). +-define(REPLY(FROM, REQUEST, RESULT), {reply, FROM, REQUEST, RESULT}). -define(EXPAND(RESULT, BATCH), [?REPLY(FROM, REQUEST, RESULT) || ?QUERY(FROM, REQUEST) <- BATCH]). -define(RESOURCE_ERROR(Reason, Msg), @@ -65,10 +65,17 @@ -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}). -type id() :: binary(). +-type query() :: {query, from(), request()}. -type request() :: term(). -type result() :: term(). -type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined. -type from() :: pid() | reply_fun(). +-type query_opts() :: #{ + %% The key used for picking a resource worker + pick_key => term() +}. + +-export_type([query_opts/0]). -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) -> {{from(), result()}, NewCbState :: term()}. @@ -78,26 +85,20 @@ callback_mode() -> [state_functions]. start_link(Id, Index, Opts) -> gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []). --spec query(id(), request()) -> Result :: term(). -query(Id, Request) -> - query(Id, self(), Request). - --spec query(id(), term(), request()) -> Result :: term(). -query(Id, PickKey, Request) -> - pick_query(call, Id, PickKey, {query, Request}). - --spec query_async(id(), request(), reply_fun()) -> Result :: term(). -query_async(Id, Request, ReplyFun) -> - query_async(Id, self(), Request, ReplyFun). - --spec query_async(id(), term(), request(), reply_fun()) -> Result :: term(). -query_async(Id, PickKey, Request, ReplyFun) -> - pick_query(cast, Id, PickKey, {query, Request, ReplyFun}). +-spec query(id(), request(), query_opts()) -> Result :: term(). +query(Id, Request, Opts) -> + PickKey = maps:get(pick_key, Opts, self()), + Timeout = maps:get(timeout, Opts, infinity), + pick_call(Id, PickKey, {query, Request}, Timeout). -spec block(pid() | atom()) -> ok. block(ServerRef) -> gen_statem:cast(ServerRef, block). +-spec block(pid() | atom(), [query()]) -> ok. +block(ServerRef, Query) -> + gen_statem:cast(ServerRef, {block, Query}). + -spec resume(pid() | atom()) -> ok. resume(ServerRef) -> gen_statem:cast(ServerRef, resume). @@ -121,6 +122,12 @@ init({Id, Index, Opts}) -> St = #{ id => Id, index => Index, + %% query_mode = dynamic | sync | async + %% TODO: + %% dynamic mode is async mode when things are going well, but becomes sync mode + %% if the resource worker is overloaded + query_mode => maps:get(query_mode, Opts, sync), + async_reply_fun => maps:get(async_reply_fun, Opts, undefined), batch_enabled => maps:get(batch_enabled, Opts, false), batch_size => BatchSize, batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), @@ -135,9 +142,11 @@ running(cast, resume, _St) -> keep_state_and_data; running(cast, block, St) -> {next_state, block, St}; -running(cast, {query, Request, ReplyFun}, St) -> - query_or_acc(ReplyFun, Request, St); -running({call, From}, {query, Request}, St) -> +running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) -> + Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]), + {next_state, block, St#{queue := Q1}}; +running({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) -> + From = maybe_quick_return(QM, From0, ReplyFun), query_or_acc(From, Request, St); running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> flush(St#{tref := undefined}); @@ -149,13 +158,15 @@ running(info, Info, _St) -> blocked(cast, block, _St) -> keep_state_and_data; +blocked(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) -> + Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]), + {keep_state, St#{queue := Q1}}; blocked(cast, resume, St) -> do_resume(St); blocked(state_timeout, resume, St) -> do_resume(St); -blocked(cast, {query, Request, ReplyFun}, St) -> - handle_blocked(ReplyFun, Request, St); -blocked({call, From}, {query, Request}, St) -> +blocked({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) -> + From = maybe_quick_return(QM, From0, ReplyFun), handle_blocked(From, Request, St). terminate(_Reason, #{id := Id, index := Index}) -> @@ -173,15 +184,23 @@ estimate_size(QItem) -> size(queue_item_marshaller(QItem)). %%============================================================================== -pick_query(Fun, Id, Key, Query) -> +maybe_quick_return(sync, From, _ReplyFun) -> + From; +maybe_quick_return(async, From, ReplyFun) -> + ok = gen_statem:reply(From), + ReplyFun. + +pick_call(Id, Key, Query, Timeout) -> try gproc_pool:pick_worker(Id, Key) of Pid when is_pid(Pid) -> - gen_statem:Fun(Pid, Query); + gen_statem:call(Pid, Query, {clean_timeout, Timeout}); _ -> ?RESOURCE_ERROR(not_created, "resource not found") catch error:badarg -> - ?RESOURCE_ERROR(not_created, "resource not found") + ?RESOURCE_ERROR(not_created, "resource not found"); + exit:{timeout, _} -> + ?RESOURCE_ERROR(timeout, "call resource timeout") end. do_resume(#{queue := undefined} = St) -> @@ -190,8 +209,8 @@ do_resume(#{queue := Q, id := Id} = St) -> case replayq:peek(Q) of empty -> {next_state, running, St}; - ?Q_ITEM(First) -> - Result = call_query(Id, First), + ?Q_ITEM(FirstQuery) -> + Result = call_query(sync, Id, FirstQuery, 1), case handle_query_result(Id, Result, false) of %% Send failed because resource down true -> @@ -206,8 +225,8 @@ do_resume(#{queue := Q, id := Id} = St) -> handle_blocked(From, Request, #{id := Id, queue := Q} = St) -> Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), - _ = reply_caller(Id, ?REPLY(From, Request, Error), false), - {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}}. + _ = reply_caller(Id, ?REPLY(From, Request, Error)), + {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}}. drop_head(Q) -> {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}), @@ -221,17 +240,18 @@ query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Lef true -> flush(St); false -> {keep_state, ensure_flush_timer(St)} end; -query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id} = St) -> - case send_query(From, Request, Id) of +query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, query_mode := QM} = St) -> + case send_query(QM, From, Request, Id) of true -> - {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Request)])}}; + Query = ?QUERY(From, Request), + {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Query)])}}; false -> {keep_state, St} end. -send_query(From, Request, Id) -> - Result = call_query(Id, Request), - reply_caller(Id, ?REPLY(From, Request, Result), false). +send_query(QM, From, Request, Id) -> + Result = call_query(QM, Id, ?QUERY(From, Request), 1), + reply_caller(Id, ?REPLY(From, Request, Result)). flush(#{acc := []} = St) -> {keep_state, St}; @@ -240,31 +260,37 @@ flush( id := Id, acc := Batch, batch_size := Size, - queue := Q0 + queue := Q0, + query_mode := QM } = St ) -> - BatchResults = maybe_expand_batch_result(call_batch_query(Id, Batch), Batch), + Result = call_query(QM, Id, Batch, length(Batch)), St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}), - case batch_reply_caller(Id, BatchResults) of + case batch_reply_caller(Id, Result, Batch) of true -> - Q1 = maybe_append_queue(Q0, [?Q_ITEM(Request) || ?QUERY(_, Request) <- Batch]), + Q1 = maybe_append_queue(Q0, [?Q_ITEM(Query) || Query <- Batch]), {next_state, blocked, St1#{queue := Q1}}; false -> {keep_state, St1} end. -maybe_append_queue(undefined, _Request) -> undefined; -maybe_append_queue(Q, Request) -> replayq:append(Q, Request). +maybe_append_queue(undefined, _Items) -> undefined; +maybe_append_queue(Q, Items) -> replayq:append(Q, Items). -batch_reply_caller(Id, BatchResults) -> +batch_reply_caller(Id, BatchResult, Batch) -> lists:foldl( fun(Reply, BlockWorker) -> reply_caller(Id, Reply, BlockWorker) end, false, - BatchResults + %% the `Mod:on_batch_query/3` returns a single result for a batch, + %% so we need to expand + ?EXPAND(BatchResult, Batch) ). +reply_caller(Id, Reply) -> + reply_caller(Id, Reply, false). + reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) -> handle_query_result(Id, Result, BlockWorker); reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) -> @@ -295,45 +321,77 @@ handle_query_result(Id, Result, BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, success), BlockWorker. -call_query(Id, Request) -> - do_call_query(on_query, Id, Request, 1). - -call_batch_query(Id, Batch) -> - do_call_query(on_batch_query, Id, Batch, length(Batch)). - -do_call_query(Fun, Id, Data, Count) -> +call_query(QM, Id, Query, QueryLen) -> case emqx_resource_manager:ets_lookup(Id) of - {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> - ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, Count), - try Mod:Fun(Id, Data, ResourceState) of - %% if the callback module (connector) wants to return an error that - %% makes the current resource goes into the `error` state, it should - %% return `{resource_down, Reason}` - Result -> Result - catch - Err:Reason:ST -> - Msg = io_lib:format( - "call query failed, func: ~s:~s/3, error: ~0p", - [Mod, Fun, {Err, Reason, ST}] - ), - ?RESOURCE_ERROR(exception, Msg) - end; + {ok, _Group, #{callback_mode := CM, mod := Mod, state := ResSt, status := connected}} -> + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, QueryLen), + apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt); {ok, _Group, #{status := stopped}} -> ?RESOURCE_ERROR(stopped, "resource stopped or disabled"); - {ok, _Group, _Data} -> + {ok, _Group, #{status := S}} when S == connecting; S == disconnected -> ?RESOURCE_ERROR(not_connected, "resource not connected"); {error, not_found} -> ?RESOURCE_ERROR(not_found, "resource not found") end. -%% the result is already expaned by the `Mod:on_query/3` -maybe_expand_batch_result(Results, _Batch) when is_list(Results) -> - Results; -%% the `Mod:on_query/3` returns a sinle result for a batch, so it is need expand -maybe_expand_batch_result(Result, Batch) -> - ?EXPAND(Result, Batch). +-define(APPLY_RESOURCE(EXPR, REQ), + try + %% if the callback module (connector) wants to return an error that + %% makes the current resource goes into the `error` state, it should + %% return `{resource_down, Reason}` + EXPR + catch + ERR:REASON:STACKTRACE -> + MSG = io_lib:format( + "call query failed, func: ~s, id: ~s, error: ~0p, Request: ~0p", + [??EXPR, Id, {ERR, REASON, STACKTRACE}, REQ], + [{chars_limit, 1024}] + ), + ?RESOURCE_ERROR(exception, MSG) + end +). + +apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request), ResSt) -> + ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request); +apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) -> + ReplyFun = fun ?MODULE:reply_after_query/4, + ?APPLY_RESOURCE( + begin + _ = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt), + ok_async + end, + Request + ); +apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) -> + Requests = [Request || ?QUERY(_From, Request) <- Batch], + ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch); +apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) -> + Requests = [Request || ?QUERY(_From, Request) <- Batch], + ReplyFun = fun ?MODULE:batch_reply_after_query/4, + ?APPLY_RESOURCE( + begin + _ = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt), + ok_async + end, + Batch + ). + +reply_after_query(Pid, Id, ?QUERY(From, Request) = Query, Result) -> + case reply_caller(Id, ?REPLY(From, Request, Result)) of + true -> ?MODULE:block(Pid, [Query]); + false -> ok + end. + +batch_reply_after_query(Pid, Id, Batch, Result) -> + case batch_reply_caller(Id, Result, Batch) of + true -> ?MODULE:block(Pid, Batch); + false -> ok + end. %%============================================================================== +call_mode(sync, _) -> sync; +call_mode(async, always_sync) -> sync; +call_mode(async, async_if_possible) -> async. is_ok_result(ok) -> true; diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index e9c77e915..740f110ec 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -22,6 +22,7 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, on_query/3, @@ -49,6 +50,8 @@ register(required) -> true; register(default) -> false; register(_) -> undefined. +callback_mode() -> always_sync. + on_start(_InstId, #{create_error := true}) -> error("some error"); on_start(InstId, #{name := Name, stop_error := true} = Opts) -> @@ -58,12 +61,6 @@ on_start(InstId, #{name := Name, stop_error := true} = Opts) -> stop_error => true, pid => spawn_counter_process(Name, Register) }}; -on_start(InstId, #{name := Name} = Opts) -> - Register = maps:get(register, Opts, false), - {ok, Opts#{ - id => InstId, - pid => spawn_counter_process(Name, Register) - }}; on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), {ok, Opts#{ From 145ff66a9add21898ee6f6805273d72cb57ab58d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 9 Aug 2022 08:58:42 +0800 Subject: [PATCH 39/71] fix: issues found by dialyzer and elvis --- apps/emqx_resource/include/emqx_resource.hrl | 3 +- apps/emqx_resource/src/emqx_resource.erl | 8 ++++- .../src/emqx_resource_manager.erl | 5 ++-- .../src/emqx_ee_connector_hstreamdb.erl | 29 +++++++++---------- .../src/emqx_ee_connector_influxdb.erl | 17 ++++++----- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index c691789c2..a59877a30 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -22,10 +22,11 @@ -type resource_spec() :: map(). -type resource_state() :: term(). -type resource_status() :: connected | disconnected | connecting | stopped. +-type callback_mode() :: always_sync | async_if_possible. -type resource_data() :: #{ id := resource_id(), mod := module(), - callback_mode := always_sync | async_if_possible, + callback_mode := callback_mode(), config := resource_config(), state := resource_state(), status := resource_status(), diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index a54a77e19..d17f4ce19 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -80,8 +80,10 @@ %% Direct calls to the callback module -%% start the instance -export([ + %% get the callback mode of a specific module + get_callback_mode/1, + %% start the instance call_start/3, %% verify if the resource is working normally call_health_check/3, @@ -285,6 +287,10 @@ generate_id(Name) when is_binary(Name) -> -spec list_group_instances(resource_group()) -> [resource_id()]. list_group_instances(Group) -> emqx_resource_manager:list_group(Group). +-spec get_callback_mode(module()) -> callback_mode(). +get_callback_mode(Mod) -> + Mod:callback_mode(). + -spec call_start(manager_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. call_start(MgrId, Mod, Config) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index d6e5a1493..3310555d1 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -54,6 +54,7 @@ % State record -record(data, {id, manager_id, group, mod, callback_mode, config, opts, status, state, error}). +-type data() :: #data{}. -define(SHORT_HEALTHCHECK_INTERVAL, 1000). -define(HEALTHCHECK_INTERVAL, 15000). @@ -259,7 +260,7 @@ start_link(MgrId, ResId, Group, ResourceType, Config, Opts) -> manager_id = MgrId, group = Group, mod = ResourceType, - callback_mode = ResourceType:callback_mode(), + callback_mode = emqx_resource:get_callback_mode(ResourceType), config = Config, opts = Opts, status = connecting, @@ -560,7 +561,7 @@ maybe_reply(Actions, undefined, _Reply) -> maybe_reply(Actions, From, Reply) -> [{reply, From, Reply} | Actions]. --spec data_record_to_external_map_with_metrics(#data{}) -> resource_data(). +-spec data_record_to_external_map_with_metrics(data()) -> resource_data(). data_record_to_external_map_with_metrics(Data) -> #{ id => Data#data.id, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl index 8ee37cd8a..3892b7fc0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl @@ -13,9 +13,10 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -33,6 +34,7 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +callback_mode() -> always_sync. on_start(InstId, Config) -> start_client(InstId, Config). @@ -52,11 +54,10 @@ on_stop(InstId, #{client := Client, producer := Producer}) -> on_query( _InstId, {send_message, Data}, - AfterQuery, #{producer := Producer, ordering_key := OrderingKey, payload := Payload} ) -> Record = to_record(OrderingKey, Payload, Data), - do_append(AfterQuery, Producer, Record). + do_append(Producer, Record). on_get_status(_InstId, #{client := Client}) -> case is_alive(Client) of @@ -260,27 +261,26 @@ to_record(OrderingKey, Payload) when is_binary(OrderingKey) -> to_record(OrderingKey, Payload) -> hstreamdb:to_record(OrderingKey, raw, Payload). -do_append(AfterQuery, Producer, Record) -> - do_append(AfterQuery, false, Producer, Record). +do_append(Producer, Record) -> + do_append(false, Producer, Record). %% TODO: this append is async, remove or change it after we have better disk cache. -% do_append(AfterQuery, true, Producer, Record) -> +% do_append(true, Producer, Record) -> % case hstreamdb:append(Producer, Record) of % ok -> % ?SLOG(debug, #{ % msg => "hstreamdb producer async append success", % record => Record -% }), -% emqx_resource:query_success(AfterQuery); -% {error, Reason} -> +% }); +% {error, Reason} = Err -> % ?SLOG(error, #{ % msg => "hstreamdb producer async append failed", % reason => Reason, % record => Record % }), -% emqx_resource:query_failed(AfterQuery) +% Err % end; -do_append(AfterQuery, false, Producer, Record) -> +do_append(false, Producer, Record) -> %% TODO: this append is sync, but it does not support [Record], can only append one Record. %% Change it after we have better dick cache. case hstreamdb:append_flush(Producer, Record) of @@ -288,15 +288,14 @@ do_append(AfterQuery, false, Producer, Record) -> ?SLOG(debug, #{ msg => "hstreamdb producer sync append success", record => Record - }), - emqx_resource:query_success(AfterQuery); - {error, Reason} -> + }); + {error, Reason} = Err -> ?SLOG(error, #{ msg => "hstreamdb producer sync append failed", reason => Reason, record => Record }), - emqx_resource:query_failed(AfterQuery) + Err end. client_name(InstId) -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 9582f1729..09b3d7350 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -13,9 +13,10 @@ %% callbacks of behaviour emqx_resource -export([ + callback_mode/0, on_start/2, on_stop/2, - on_query/4, + on_query/3, on_get_status/2 ]). @@ -28,6 +29,7 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +callback_mode() -> always_sync. on_start(InstId, Config) -> start_client(InstId, Config). @@ -35,8 +37,8 @@ on_start(InstId, Config) -> on_stop(_InstId, #{client := Client}) -> influxdb:stop_client(Client). -on_query(InstId, {send_message, Data}, AfterQuery, State) -> - do_query(InstId, {send_message, Data}, AfterQuery, State). +on_query(InstId, {send_message, Data}, State) -> + do_query(InstId, {send_message, Data}, State). on_get_status(_InstId, #{client := Client}) -> case influxdb:is_alive(Client) of @@ -308,7 +310,7 @@ ssl_config(SSL = #{enable := true}) -> %% ------------------------------------------------------------------------------------------------- %% Query -do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client}) -> +do_query(InstId, {send_message, Data}, State = #{client := Client}) -> {Points, Errs} = data_to_points(Data, State), lists:foreach( fun({error, Reason}) -> @@ -326,15 +328,14 @@ do_query(InstId, {send_message, Data}, AfterQuery, State = #{client := Client}) msg => "influxdb write point success", connector => InstId, points => Points - }), - emqx_resource:query_success(AfterQuery); - {error, Reason} -> + }); + {error, Reason} = Err -> ?SLOG(error, #{ msg => "influxdb write point failed", connector => InstId, reason => Reason }), - emqx_resource:query_failed(AfterQuery) + Err end. %% ------------------------------------------------------------------------------------------------- From efd6c56dd9554a05b8b0772856008cdae40ab6ad Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 9 Aug 2022 13:00:21 +0800 Subject: [PATCH 40/71] fix: test cases for batch query sync --- .../src/emqx_resource_worker.erl | 7 +++- .../test/emqx_connector_demo.erl | 8 ++--- .../test/emqx_resource_SUITE.erl | 35 +++++++++++++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 2115fed86..e20345c2b 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -22,6 +22,7 @@ -include("emqx_resource.hrl"). -include("emqx_resource_utils.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -behaviour(gen_statem). @@ -351,9 +352,11 @@ call_query(QM, Id, Query, QueryLen) -> end ). -apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request), ResSt) -> +apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request) = _Query, ResSt) -> + ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}), ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request); apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) -> + ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}), ReplyFun = fun ?MODULE:reply_after_query/4, ?APPLY_RESOURCE( begin @@ -363,9 +366,11 @@ apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) -> Request ); apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) -> + ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}), Requests = [Request || ?QUERY(_From, Request) <- Batch], ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch); apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) -> + ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}), Requests = [Request || ?QUERY(_From, Request) <- Batch], ReplyFun = fun ?MODULE:batch_reply_after_query/4, ?APPLY_RESOURCE( diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 740f110ec..40734de68 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -94,18 +94,18 @@ on_query(_InstId, get_counter, #{pid := Pid}) -> on_batch_query(InstId, BatchReq, State) -> %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed. case hd(BatchReq) of - {_From, {inc_counter, _}} -> + {inc_counter, _} -> batch_inc_counter(InstId, BatchReq, State); - {_From, get_counter} -> + get_counter -> batch_get_counter(InstId, State) end. batch_inc_counter(InstId, BatchReq, State) -> TotalN = lists:foldl( fun - ({_From, {inc_counter, N}}, Total) -> + ({inc_counter, N}, Total) -> Total + N; - ({_From, Req}, _Total) -> + (Req, _Total) -> error({mixed_requests_not_allowed, {inc_counter, Req}}) end, 0, diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 4c5a36f4b..fb2bdfd7c 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include("emqx_resource.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(TEST_RESOURCE, emqx_connector_demo). -define(ID, <<"id">>). @@ -207,17 +208,40 @@ t_batch_query_counter(_) -> ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, - #{name => test_resource, register => true, batch_enabled => true} + #{name => test_resource, register => true}, + #{batch_enabled => true} ), - {ok, 0} = emqx_resource:query(?ID, get_counter), + ?check_trace( + #{timetrap => 10000, timeout => 1000}, + emqx_resource:query(?ID, get_counter), + fun(Result, Trace) -> + ?assertMatch({ok, 0}, Result), + QueryTrace = ?of_kind(call_batch_query, Trace), + ?assertMatch([#{batch := [{query, _, get_counter}]}], QueryTrace) + end + ), + + ?check_trace( + #{timetrap => 10000, timeout => 1000}, + inc_counter_in_parallel(1000), + fun(Trace) -> + QueryTrace = ?of_kind(call_batch_query, Trace), + ?assertMatch([#{batch := BatchReq} | _] when length(BatchReq) > 1, QueryTrace) + end + ), + {ok, 1000} = emqx_resource:query(?ID, get_counter), + + ok = emqx_resource:remove_local(?ID). + +inc_counter_in_parallel(N) -> Parent = self(), Pids = [ erlang:spawn(fun() -> ok = emqx_resource:query(?ID, {inc_counter, 1}), Parent ! {complete, self()} end) - || _ <- lists:seq(1, 1000) + || _ <- lists:seq(1, N) ], [ receive @@ -226,10 +250,7 @@ t_batch_query_counter(_) -> ct:fail({wait_for_query_timeout, Pid}) end || Pid <- Pids - ], - {ok, 1000} = emqx_resource:query(?ID, get_counter), - - ok = emqx_resource:remove_local(?ID). + ]. t_healthy_timeout(_) -> {ok, _} = emqx_resource:create_local( From 82550a585a4a0ce7a3e72733d67d683c03f00645 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 10 Aug 2022 00:30:42 +0800 Subject: [PATCH 41/71] fix: add test cases for query async --- apps/emqx_resource/include/emqx_resource.hrl | 6 + apps/emqx_resource/src/emqx_resource.erl | 13 +- .../src/emqx_resource_worker.erl | 48 +++++-- .../test/emqx_connector_demo.erl | 17 ++- .../test/emqx_resource_SUITE.erl | 122 +++++++++++++++--- 5 files changed, 171 insertions(+), 35 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index a59877a30..75cba14ad 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -23,6 +23,12 @@ -type resource_state() :: term(). -type resource_status() :: connected | disconnected | connecting | stopped. -type callback_mode() :: always_sync | async_if_possible. +-type result() :: term(). +-type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined. +-type query_opts() :: #{ + %% The key used for picking a resource worker + pick_key => term() +}. -type resource_data() :: #{ id := resource_id(), mod := module(), diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index d17f4ce19..f3f2d5fb9 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -75,7 +75,10 @@ %% stop the instance stop/1, %% query the instance - query/2 + query/2, + %% query the instance without batching and queuing messages. + simple_sync_query/2, + simple_async_query/3 ]). %% Direct calls to the callback module @@ -232,6 +235,14 @@ query(ResId, Request) -> query(ResId, Request, Opts) -> emqx_resource_worker:query(ResId, Request, Opts). +-spec simple_sync_query(resource_id(), Request :: term()) -> Result :: term(). +simple_sync_query(ResId, Request) -> + emqx_resource_worker:simple_sync_query(ResId, Request). + +-spec simple_async_query(resource_id(), Request :: term(), reply_fun()) -> Result :: term(). +simple_async_query(ResId, Request, ReplyFun) -> + emqx_resource_worker:simple_async_query(ResId, Request, ReplyFun). + -spec start(resource_id()) -> ok | {error, Reason :: term()}. start(ResId) -> start(ResId, #{}). diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index e20345c2b..d19353a29 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -34,6 +34,11 @@ resume/1 ]). +-export([ + simple_sync_query/2, + simple_async_query/3 +]). + -export([ callback_mode/0, init/1, @@ -68,13 +73,7 @@ -type id() :: binary(). -type query() :: {query, from(), request()}. -type request() :: term(). --type result() :: term(). --type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined. -type from() :: pid() | reply_fun(). --type query_opts() :: #{ - %% The key used for picking a resource worker - pick_key => term() -}. -export_type([query_opts/0]). @@ -92,6 +91,19 @@ query(Id, Request, Opts) -> Timeout = maps:get(timeout, Opts, infinity), pick_call(Id, PickKey, {query, Request}, Timeout). +%% simple query the resource without batching and queuing messages. +-spec simple_sync_query(id(), request()) -> Result :: term(). +simple_sync_query(Id, Request) -> + Result = call_query(sync, Id, ?QUERY(self(), Request), 1), + _ = handle_query_result(Id, Result, false), + Result. + +-spec simple_async_query(id(), request(), reply_fun()) -> Result :: term(). +simple_async_query(Id, Request, ReplyFun) -> + Result = call_query(async, Id, ?QUERY(ReplyFun, Request), 1), + _ = handle_query_result(Id, Result, false), + Result. + -spec block(pid() | atom()) -> ok. block(ServerRef) -> gen_statem:cast(ServerRef, block). @@ -188,7 +200,7 @@ estimate_size(QItem) -> maybe_quick_return(sync, From, _ReplyFun) -> From; maybe_quick_return(async, From, ReplyFun) -> - ok = gen_statem:reply(From), + gen_statem:reply(From, ok), ReplyFun. pick_call(Id, Key, Query, Timeout) -> @@ -295,7 +307,11 @@ reply_caller(Id, Reply) -> reply_caller(Id, ?REPLY(undefined, _, Result), BlockWorker) -> handle_query_result(Id, Result, BlockWorker); reply_caller(Id, ?REPLY({ReplyFun, Args}, _, Result), BlockWorker) when is_function(ReplyFun) -> - ?SAFE_CALL(ReplyFun(Result, Args)), + _ = + case Result of + {async_return, _} -> ok; + _ -> apply(ReplyFun, Args ++ [Result]) + end, handle_query_result(Id, Result, BlockWorker); reply_caller(Id, ?REPLY(From, _, Result), BlockWorker) -> gen_statem:reply(From, Result), @@ -316,6 +332,10 @@ handle_query_result(Id, {error, _}, BlockWorker) -> handle_query_result(Id, {resource_down, _}, _BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down), true; +handle_query_result(_Id, {async_return, {resource_down, _}}, _BlockWorker) -> + true; +handle_query_result(_Id, {async_return, ok}, BlockWorker) -> + BlockWorker; handle_query_result(Id, Result, BlockWorker) -> %% assert true = is_ok_result(Result), @@ -352,16 +372,16 @@ call_query(QM, Id, Query, QueryLen) -> end ). -apply_query_fun(sync, Mod, Id, ?QUERY(_From, Request) = _Query, ResSt) -> +apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt) -> ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}), ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request); -apply_query_fun(async, Mod, Id, ?QUERY(_From, Request) = Query, ResSt) -> +apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt) -> ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}), ReplyFun = fun ?MODULE:reply_after_query/4, ?APPLY_RESOURCE( begin - _ = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt), - ok_async + Result = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt), + {async_return, Result} end, Request ); @@ -375,8 +395,8 @@ apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) -> ReplyFun = fun ?MODULE:batch_reply_after_query/4, ?APPLY_RESOURCE( begin - _ = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt), - ok_async + Result = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt), + {async_return, Result} end, Batch ). diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 40734de68..3bea71993 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -26,11 +26,12 @@ on_start/2, on_stop/2, on_query/3, + on_query_async/4, on_batch_query/3, on_get_status/2 ]). --export([counter_loop/1]). +-export([counter_loop/1, set_callback_mode/1]). %% callbacks for emqx_resource config schema -export([roots/0]). @@ -50,7 +51,12 @@ register(required) -> true; register(default) -> false; register(_) -> undefined. -callback_mode() -> always_sync. +-define(CM_KEY, {?MODULE, callback_mode}). +callback_mode() -> + persistent_term:get(?CM_KEY). + +set_callback_mode(Mode) -> + persistent_term:put(?CM_KEY, Mode). on_start(_InstId, #{create_error := true}) -> error("some error"); @@ -91,6 +97,10 @@ on_query(_InstId, get_counter, #{pid := Pid}) -> {error, timeout} end. +on_query_async(_InstId, Query, ReplyFun, State) -> + Result = on_query(_InstId, Query, State), + apply_reply(ReplyFun, Result). + on_batch_query(InstId, BatchReq, State) -> %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed. case hd(BatchReq) of @@ -147,3 +157,6 @@ maybe_register(Name, Pid, true) -> erlang:register(Name, Pid); maybe_register(_Name, _Pid, false) -> true. + +apply_reply({ReplyFun, Args}, Result) when is_function(ReplyFun) -> + apply(ReplyFun, Args ++ [Result]). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index fb2bdfd7c..5177e792c 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -28,6 +28,7 @@ -define(ID, <<"id">>). -define(DEFAULT_RESOURCE_GROUP, <<"default">>). -define(RESOURCE_ERROR(REASON), {error, {resource_error, #{reason := REASON}}}). +-define(TRACE_OPTS, #{timetrap => 10000, timeout => 1000}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -36,6 +37,7 @@ groups() -> []. init_per_testcase(_, Config) -> + emqx_connector_demo:set_callback_mode(always_sync), Config. end_per_testcase(_, _Config) -> _ = emqx_resource:remove(?ID). @@ -213,7 +215,7 @@ t_batch_query_counter(_) -> ), ?check_trace( - #{timetrap => 10000, timeout => 1000}, + ?TRACE_OPTS, emqx_resource:query(?ID, get_counter), fun(Result, Trace) -> ?assertMatch({ok, 0}, Result), @@ -223,7 +225,7 @@ t_batch_query_counter(_) -> ), ?check_trace( - #{timetrap => 10000, timeout => 1000}, + ?TRACE_OPTS, inc_counter_in_parallel(1000), fun(Trace) -> QueryTrace = ?of_kind(call_batch_query, Trace), @@ -234,23 +236,90 @@ t_batch_query_counter(_) -> ok = emqx_resource:remove_local(?ID). -inc_counter_in_parallel(N) -> - Parent = self(), - Pids = [ - erlang:spawn(fun() -> - ok = emqx_resource:query(?ID, {inc_counter, 1}), - Parent ! {complete, self()} - end) - || _ <- lists:seq(1, N) - ], - [ - receive - {complete, Pid} -> ok - after 1000 -> - ct:fail({wait_for_query_timeout, Pid}) +t_query_counter_async(_) -> + {ok, _} = emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, register => true}, + #{query_mode => async} + ), + ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), + ?check_trace( + ?TRACE_OPTS, + inc_counter_in_parallel(1000), + fun(Trace) -> + %% the callback_mode if 'emqx_connector_demo' is 'always_sync'. + QueryTrace = ?of_kind(call_query, Trace), + ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace) end - || Pid <- Pids - ]. + ), + %% wait for 1s to make sure all the aysnc query is sent to the resource. + timer:sleep(1000), + %% simple query ignores the query_mode and batching settings in the resource_worker + ?check_trace( + ?TRACE_OPTS, + emqx_resource:simple_sync_query(?ID, get_counter), + fun(Result, Trace) -> + ?assertMatch({ok, 1000}, Result), + %% the callback_mode if 'emqx_connector_demo' is 'always_sync'. + QueryTrace = ?of_kind(call_query, Trace), + ?assertMatch([#{query := {query, _, get_counter}}], QueryTrace) + end + ), + {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), + ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C), + ok = emqx_resource:remove_local(?ID). + +t_query_counter_async_2(_) -> + emqx_connector_demo:set_callback_mode(async_if_possible), + + Tab0 = ets:new(?FUNCTION_NAME, [bag, public]), + Insert = fun(Tab, Result) -> + ets:insert(Tab, {make_ref(), Result}) + end, + {ok, _} = emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, register => true}, + #{query_mode => async, async_reply_fun => {Insert, [Tab0]}} + ), + ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), + ?check_trace( + ?TRACE_OPTS, + inc_counter_in_parallel(1000), + fun(Trace) -> + QueryTrace = ?of_kind(call_query_async, Trace), + ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace) + end + ), + + %% wait for 1s to make sure all the aysnc query is sent to the resource. + timer:sleep(1000), + %% simple query ignores the query_mode and batching settings in the resource_worker + ?check_trace( + ?TRACE_OPTS, + emqx_resource:simple_sync_query(?ID, get_counter), + fun(Result, Trace) -> + ?assertMatch({ok, 1000}, Result), + QueryTrace = ?of_kind(call_query, Trace), + ?assertMatch([#{query := {query, _, get_counter}}], QueryTrace) + end + ), + {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), + ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C), + ?assertMatch(1000, ets:info(Tab0, size)), + ?assert( + lists:all( + fun + ({_, ok}) -> true; + (_) -> false + end, + ets:tab2list(Tab0) + ) + ), + ok = emqx_resource:remove_local(?ID). t_healthy_timeout(_) -> {ok, _} = emqx_resource:create_local( @@ -480,6 +549,23 @@ t_auto_retry(_) -> %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ +inc_counter_in_parallel(N) -> + Parent = self(), + Pids = [ + erlang:spawn(fun() -> + emqx_resource:query(?ID, {inc_counter, 1}), + Parent ! {complete, self()} + end) + || _ <- lists:seq(1, N) + ], + [ + receive + {complete, Pid} -> ok + after 1000 -> + ct:fail({wait_for_query_timeout, Pid}) + end + || Pid <- Pids + ]. bin_config() -> <<"\"name\": \"test_resource\"">>. From 07933e36389587d57a656c10a6fee70667388c0e Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 10 Aug 2022 17:39:24 +0800 Subject: [PATCH 42/71] feat: translate map to object in schema.json --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf.erl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 1441a4180..854fdac07 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index bc8b52702..ce54695a5 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -399,6 +399,10 @@ typename_to_spec("failure_strategy()", _Mod) -> #{type => enum, symbols => [force, drop, throw]}; typename_to_spec("initial()", _Mod) -> #{type => string}; +typename_to_spec("map()", _Mod) -> + #{type => object}; +typename_to_spec("#{" ++ _, Mod) -> + typename_to_spec("map()", Mod); typename_to_spec(Name, Mod) -> Spec = range(Name), Spec1 = remote_module_type(Spec, Name, Mod), From 6203a01320251d5fd93c938ccfab64cb01988b86 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 11 Aug 2022 00:29:50 +0800 Subject: [PATCH 43/71] feat: add inflight window to emqx_resource --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_resource/include/emqx_resource.hrl | 3 +- apps/emqx_resource/src/emqx_resource.erl | 1 + .../src/emqx_resource_worker.erl | 224 +++++++++++++----- .../test/emqx_connector_demo.erl | 56 ++++- .../test/emqx_resource_SUITE.erl | 136 ++++++++++- 6 files changed, 341 insertions(+), 81 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index fe19ed066..3be8d38fd 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "An OTP application"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 75cba14ad..5c561a8d3 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -27,7 +27,8 @@ -type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined. -type query_opts() :: #{ %% The key used for picking a resource worker - pick_key => term() + pick_key => term(), + async_reply_fun => reply_fun() }. -type resource_data() :: #{ id := resource_id(), diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index f3f2d5fb9..0d2289696 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -76,6 +76,7 @@ stop/1, %% query the instance query/2, + query/3, %% query the instance without batching and queuing messages. simple_sync_query/2, simple_async_query/3 diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index d19353a29..716842bf9 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -50,7 +50,7 @@ -export([queue_item_marshaller/1, estimate_size/1]). --export([reply_after_query/4, batch_reply_after_query/4]). +-export([reply_after_query/6, batch_reply_after_query/6]). -define(RESUME_INTERVAL, 15000). @@ -69,18 +69,18 @@ {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}} ). -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}). +-define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024). +-define(DEFAULT_INFLIGHT, 100). -type id() :: binary(). -type query() :: {query, from(), request()}. -type request() :: term(). -type from() :: pid() | reply_fun(). --export_type([query_opts/0]). - -callback batcher_flush(Acc :: [{from(), request()}], CbState :: term()) -> {{from(), result()}, NewCbState :: term()}. -callback_mode() -> [state_functions]. +callback_mode() -> [state_functions, state_enter]. start_link(Id, Index, Opts) -> gen_statem:start_link({local, name(Id, Index)}, ?MODULE, {Id, Index, Opts}, []). @@ -89,18 +89,18 @@ start_link(Id, Index, Opts) -> query(Id, Request, Opts) -> PickKey = maps:get(pick_key, Opts, self()), Timeout = maps:get(timeout, Opts, infinity), - pick_call(Id, PickKey, {query, Request}, Timeout). + pick_call(Id, PickKey, {query, Request, Opts}, Timeout). %% simple query the resource without batching and queuing messages. -spec simple_sync_query(id(), request()) -> Result :: term(). simple_sync_query(Id, Request) -> - Result = call_query(sync, Id, ?QUERY(self(), Request), 1), + Result = call_query(sync, Id, ?QUERY(self(), Request), #{}), _ = handle_query_result(Id, Result, false), Result. -spec simple_async_query(id(), request(), reply_fun()) -> Result :: term(). simple_async_query(Id, Request, ReplyFun) -> - Result = call_query(async, Id, ?QUERY(ReplyFun, Request), 1), + Result = call_query(async, Id, ?QUERY(ReplyFun, Request), #{}), _ = handle_query_result(Id, Result, false), Result. @@ -119,38 +119,44 @@ resume(ServerRef) -> init({Id, Index, Opts}) -> process_flag(trap_exit, true), true = gproc_pool:connect_worker(Id, {Id, Index}), + Name = name(Id, Index), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), Queue = case maps:get(queue_enabled, Opts, false) of true -> replayq:open(#{ dir => disk_queue_dir(Id, Index), - seg_bytes => 10000000, + seg_bytes => maps:get(queue_max_bytes, Opts, ?DEFAULT_QUEUE_SIZE), sizer => fun ?MODULE:estimate_size/1, marshaller => fun ?MODULE:queue_item_marshaller/1 }); false -> undefined end, + ok = inflight_new(Name), St = #{ id => Id, index => Index, + name => Name, %% query_mode = dynamic | sync | async %% TODO: %% dynamic mode is async mode when things are going well, but becomes sync mode %% if the resource worker is overloaded query_mode => maps:get(query_mode, Opts, sync), - async_reply_fun => maps:get(async_reply_fun, Opts, undefined), + async_inflight_window => maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT), batch_enabled => maps:get(batch_enabled, Opts, false), batch_size => BatchSize, batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), queue => Queue, + resume_interval => maps:get(resume_interval, Opts, ?RESUME_INTERVAL), acc => [], acc_left => BatchSize, tref => undefined }, {ok, blocked, St, {next_event, cast, resume}}. +running(enter, _, _St) -> + keep_state_and_data; running(cast, resume, _St) -> keep_state_and_data; running(cast, block, St) -> @@ -158,8 +164,8 @@ running(cast, block, St) -> running(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) -> Q1 = maybe_append_queue(Q, [?Q_ITEM(Query) || Query <- Batch]), {next_state, block, St#{queue := Q1}}; -running({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) -> - From = maybe_quick_return(QM, From0, ReplyFun), +running({call, From0}, {query, Request, Opts}, #{query_mode := QM} = St) -> + From = maybe_quick_return(QM, From0, maps:get(async_reply_fun, Opts, undefined)), query_or_acc(From, Request, St); running(info, {flush, Ref}, St = #{tref := {_TRef, Ref}}) -> flush(St#{tref := undefined}); @@ -169,6 +175,8 @@ running(info, Info, _St) -> ?SLOG(error, #{msg => unexpected_msg, info => Info}), keep_state_and_data. +blocked(enter, _, #{resume_interval := ResumeT} = _St) -> + {keep_state_and_data, {state_timeout, ResumeT, resume}}; blocked(cast, block, _St) -> keep_state_and_data; blocked(cast, {block, [?QUERY(_, _) | _] = Batch}, #{queue := Q} = St) when is_list(Batch) -> @@ -178,9 +186,11 @@ blocked(cast, resume, St) -> do_resume(St); blocked(state_timeout, resume, St) -> do_resume(St); -blocked({call, From0}, {query, Request}, #{query_mode := QM, async_reply_fun := ReplyFun} = St) -> - From = maybe_quick_return(QM, From0, ReplyFun), - handle_blocked(From, Request, St). +blocked({call, From0}, {query, Request, Opts}, #{id := Id, queue := Q, query_mode := QM} = St) -> + From = maybe_quick_return(QM, From0, maps:get(async_reply_fun, Opts, undefined)), + Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), + _ = reply_caller(Id, ?REPLY(From, Request, Error)), + {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}}. terminate(_Reason, #{id := Id, index := Index}) -> gproc_pool:disconnect_worker(Id, {Id, Index}). @@ -216,30 +226,44 @@ pick_call(Id, Key, Query, Timeout) -> ?RESOURCE_ERROR(timeout, "call resource timeout") end. -do_resume(#{queue := undefined} = St) -> +do_resume(#{queue := Q, id := Id, name := Name} = St) -> + case inflight_get_first(Name) of + empty -> + retry_first_from_queue(Q, Id, St); + {Ref, FirstQuery} -> + retry_first_sync(Id, FirstQuery, Name, Ref, undefined, St) + end. + +retry_first_from_queue(undefined, _Id, St) -> {next_state, running, St}; -do_resume(#{queue := Q, id := Id} = St) -> +retry_first_from_queue(Q, Id, St) -> case replayq:peek(Q) of empty -> {next_state, running, St}; ?Q_ITEM(FirstQuery) -> - Result = call_query(sync, Id, FirstQuery, 1), - case handle_query_result(Id, Result, false) of - %% Send failed because resource down - true -> - {keep_state, St, {state_timeout, ?RESUME_INTERVAL, resume}}; - %% Send ok or failed but the resource is working - false -> - %% We Send 'resume' to the end of the mailbox to give the worker - %% a chance to process 'query' requests. - {keep_state, St#{queue => drop_head(Q)}, {state_timeout, 0, resume}} - end + retry_first_sync(Id, FirstQuery, undefined, undefined, Q, St) end. -handle_blocked(From, Request, #{id := Id, queue := Q} = St) -> - Error = ?RESOURCE_ERROR(blocked, "resource is blocked"), - _ = reply_caller(Id, ?REPLY(From, Request, Error)), - {keep_state, St#{queue := maybe_append_queue(Q, [?Q_ITEM(?QUERY(From, Request))])}}. +retry_first_sync(Id, FirstQuery, Name, Ref, Q, #{resume_interval := ResumeT} = St0) -> + Result = call_query(sync, Id, FirstQuery, #{}), + case handle_query_result(Id, Result, false) of + %% Send failed because resource down + true -> + {keep_state, St0, {state_timeout, ResumeT, resume}}; + %% Send ok or failed but the resource is working + false -> + %% We Send 'resume' to the end of the mailbox to give the worker + %% a chance to process 'query' requests. + St = + case Q of + undefined -> + inflight_drop(Name, Ref), + St0; + _ -> + St0#{queue => drop_head(Q)} + end, + {keep_state, St, {state_timeout, 0, resume}} + end. drop_head(Q) -> {Q1, AckRef, _} = replayq:pop(Q, #{count_limit => 1}), @@ -254,7 +278,11 @@ query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Lef false -> {keep_state, ensure_flush_timer(St)} end; query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, query_mode := QM} = St) -> - case send_query(QM, From, Request, Id) of + QueryOpts = #{ + inflight_name => maps:get(name, St), + inflight_window => maps:get(async_inflight_window, St) + }, + case send_query(QM, From, Request, Id, QueryOpts) of true -> Query = ?QUERY(From, Request), {next_state, blocked, St#{queue := maybe_append_queue(Q, [?Q_ITEM(Query)])}}; @@ -262,8 +290,8 @@ query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, quer {keep_state, St} end. -send_query(QM, From, Request, Id) -> - Result = call_query(QM, Id, ?QUERY(From, Request), 1), +send_query(QM, From, Request, Id, QueryOpts) -> + Result = call_query(QM, Id, ?QUERY(From, Request), QueryOpts), reply_caller(Id, ?REPLY(From, Request, Result)). flush(#{acc := []} = St) -> @@ -277,7 +305,11 @@ flush( query_mode := QM } = St ) -> - Result = call_query(QM, Id, Batch, length(Batch)), + QueryOpts = #{ + inflight_name => maps:get(name, St), + inflight_window => maps:get(async_inflight_window, St) + }, + Result = call_query(QM, Id, Batch, QueryOpts), St1 = cancel_flush_timer(St#{acc_left := Size, acc := []}), case batch_reply_caller(Id, Result, Batch) of true -> @@ -332,21 +364,21 @@ handle_query_result(Id, {error, _}, BlockWorker) -> handle_query_result(Id, {resource_down, _}, _BlockWorker) -> emqx_metrics_worker:inc(?RES_METRICS, Id, resource_down), true; +handle_query_result(_Id, {async_return, inflight_full}, _BlockWorker) -> + true; handle_query_result(_Id, {async_return, {resource_down, _}}, _BlockWorker) -> true; handle_query_result(_Id, {async_return, ok}, BlockWorker) -> BlockWorker; handle_query_result(Id, Result, BlockWorker) -> - %% assert - true = is_ok_result(Result), + assert_ok_result(Result), emqx_metrics_worker:inc(?RES_METRICS, Id, success), BlockWorker. -call_query(QM, Id, Query, QueryLen) -> +call_query(QM, Id, Query, QueryOpts) -> case emqx_resource_manager:ets_lookup(Id) of {ok, _Group, #{callback_mode := CM, mod := Mod, state := ResSt, status := connected}} -> - ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, QueryLen), - apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt); + apply_query_fun(call_mode(QM, CM), Mod, Id, Query, ResSt, QueryOpts); {ok, _Group, #{status := stopped}} -> ?RESOURCE_ERROR(stopped, "resource stopped or disabled"); {ok, _Group, #{status := S}} when S == connecting; S == disconnected -> @@ -372,58 +404,119 @@ call_query(QM, Id, Query, QueryLen) -> end ). -apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt) -> +apply_query_fun(sync, Mod, Id, ?QUERY(_, Request) = _Query, ResSt, _QueryOpts) -> ?tp(call_query, #{id => Id, mod => Mod, query => _Query, res_st => ResSt}), + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched), ?APPLY_RESOURCE(Mod:on_query(Id, Request, ResSt), Request); -apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt) -> +apply_query_fun(async, Mod, Id, ?QUERY(_, Request) = Query, ResSt, QueryOpts) -> ?tp(call_query_async, #{id => Id, mod => Mod, query => Query, res_st => ResSt}), - ReplyFun = fun ?MODULE:reply_after_query/4, + Name = maps:get(inflight_name, QueryOpts, undefined), + WinSize = maps:get(inflight_window, QueryOpts, undefined), ?APPLY_RESOURCE( - begin - Result = Mod:on_query_async(Id, Request, {ReplyFun, [self(), Id, Query]}, ResSt), - {async_return, Result} + case inflight_is_full(Name, WinSize) of + true -> + ?tp(inflight_full, #{id => Id, wind_size => WinSize}), + {async_return, inflight_full}; + false -> + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched), + ReplyFun = fun ?MODULE:reply_after_query/6, + Ref = make_message_ref(), + Args = [self(), Id, Name, Ref, Query], + ok = inflight_append(Name, Ref, Query), + Result = Mod:on_query_async(Id, Request, {ReplyFun, Args}, ResSt), + {async_return, Result} end, Request ); -apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) -> +apply_query_fun(sync, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, _QueryOpts) -> ?tp(call_batch_query, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}), Requests = [Request || ?QUERY(_From, Request) <- Batch], + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)), ?APPLY_RESOURCE(Mod:on_batch_query(Id, Requests, ResSt), Batch); -apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt) -> +apply_query_fun(async, Mod, Id, [?QUERY(_, _) | _] = Batch, ResSt, QueryOpts) -> ?tp(call_batch_query_async, #{id => Id, mod => Mod, batch => Batch, res_st => ResSt}), - Requests = [Request || ?QUERY(_From, Request) <- Batch], - ReplyFun = fun ?MODULE:batch_reply_after_query/4, + Name = maps:get(inflight_name, QueryOpts, undefined), + WinSize = maps:get(inflight_window, QueryOpts, undefined), ?APPLY_RESOURCE( - begin - Result = Mod:on_batch_query_async(Id, Requests, {ReplyFun, [self(), Id, Batch]}, ResSt), - {async_return, Result} + case inflight_is_full(Name, WinSize) of + true -> + ?tp(inflight_full, #{id => Id, wind_size => WinSize}), + {async_return, inflight_full}; + false -> + ok = emqx_metrics_worker:inc(?RES_METRICS, Id, matched, length(Batch)), + ReplyFun = fun ?MODULE:batch_reply_after_query/6, + Ref = make_message_ref(), + Args = {ReplyFun, [self(), Id, Name, Ref, Batch]}, + Requests = [Request || ?QUERY(_From, Request) <- Batch], + ok = inflight_append(Name, Ref, Batch), + Result = Mod:on_batch_query_async(Id, Requests, Args, ResSt), + {async_return, Result} end, Batch ). -reply_after_query(Pid, Id, ?QUERY(From, Request) = Query, Result) -> +reply_after_query(Pid, Id, Name, Ref, ?QUERY(From, Request), Result) -> case reply_caller(Id, ?REPLY(From, Request, Result)) of - true -> ?MODULE:block(Pid, [Query]); - false -> ok + true -> ?MODULE:block(Pid); + false -> inflight_drop(Name, Ref) end. -batch_reply_after_query(Pid, Id, Batch, Result) -> +batch_reply_after_query(Pid, Id, Name, Ref, Batch, Result) -> case batch_reply_caller(Id, Result, Batch) of - true -> ?MODULE:block(Pid, Batch); - false -> ok + true -> ?MODULE:block(Pid); + false -> inflight_drop(Name, Ref) end. +%%============================================================================== +%% the inflight queue for async query + +inflight_new(Name) -> + _ = ets:new(Name, [named_table, ordered_set, public, {write_concurrency, true}]), + ok. + +inflight_get_first(Name) -> + case ets:first(Name) of + '$end_of_table' -> + empty; + Ref -> + case ets:lookup(Name, Ref) of + [Object] -> Object; + [] -> inflight_get_first(Name) + end + end. + +inflight_is_full(undefined, _) -> + false; +inflight_is_full(Name, MaxSize) -> + case ets:info(Name, size) of + Size when Size >= MaxSize -> true; + _ -> false + end. + +inflight_append(undefined, _Ref, _Query) -> + ok; +inflight_append(Name, Ref, Query) -> + ets:insert(Name, {Ref, Query}), + ok. + +inflight_drop(undefined, _) -> + ok; +inflight_drop(Name, Ref) -> + ets:delete(Name, Ref), + ok. %%============================================================================== call_mode(sync, _) -> sync; call_mode(async, always_sync) -> sync; call_mode(async, async_if_possible) -> async. -is_ok_result(ok) -> +assert_ok_result(ok) -> true; -is_ok_result(R) when is_tuple(R) -> - erlang:element(1, R) == ok; -is_ok_result(_) -> - false. +assert_ok_result({async_return, R}) -> + assert_ok_result(R); +assert_ok_result(R) when is_tuple(R) -> + ok = erlang:element(1, R); +assert_ok_result(R) -> + error({not_ok_result, R}). -spec name(id(), integer()) -> atom(). name(Id, Index) -> @@ -447,3 +540,6 @@ cancel_flush_timer(St = #{tref := undefined}) -> cancel_flush_timer(St = #{tref := {TRef, _Ref}}) -> _ = erlang:cancel_timer(TRef), St#{tref => undefined}. + +make_message_ref() -> + erlang:unique_integer([monotonic, positive]). diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 3bea71993..6e7bca18a 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -31,7 +31,7 @@ on_get_status/2 ]). --export([counter_loop/1, set_callback_mode/1]). +-export([counter_loop/0, set_callback_mode/1]). %% callbacks for emqx_resource config schema -export([roots/0]). @@ -84,9 +84,22 @@ on_query(_InstId, get_state, State) -> {ok, State}; on_query(_InstId, get_state_failed, State) -> {error, State}; -on_query(_InstId, {inc_counter, N}, #{pid := Pid}) -> - Pid ! {inc, N}, +on_query(_InstId, block, #{pid := Pid}) -> + Pid ! block, ok; +on_query(_InstId, resume, #{pid := Pid}) -> + Pid ! resume, + ok; +on_query(_InstId, {inc_counter, N}, #{pid := Pid}) -> + ReqRef = make_ref(), + From = {self(), ReqRef}, + Pid ! {From, {inc, N}}, + receive + {ReqRef, ok} -> ok; + {ReqRef, incorrect_status} -> {resource_down, incorrect_status} + after 1000 -> + {error, timeout} + end; on_query(_InstId, get_counter, #{pid := Pid}) -> ReqRef = make_ref(), From = {self(), ReqRef}, @@ -97,9 +110,12 @@ on_query(_InstId, get_counter, #{pid := Pid}) -> {error, timeout} end. -on_query_async(_InstId, Query, ReplyFun, State) -> - Result = on_query(_InstId, Query, State), - apply_reply(ReplyFun, Result). +on_query_async(_InstId, {inc_counter, N}, ReplyFun, #{pid := Pid}) -> + Pid ! {inc, N, ReplyFun}, + ok; +on_query_async(_InstId, get_counter, ReplyFun, #{pid := Pid}) -> + Pid ! {get, ReplyFun}, + ok. on_batch_query(InstId, BatchReq, State) -> %% Requests can be either 'get_counter' or 'inc_counter', but cannot be mixed. @@ -136,15 +152,35 @@ on_get_status(_InstId, #{pid := Pid}) -> end. spawn_counter_process(Name, Register) -> - Pid = spawn_link(?MODULE, counter_loop, [#{counter => 0}]), + Pid = spawn_link(?MODULE, counter_loop, []), true = maybe_register(Name, Pid, Register), Pid. -counter_loop(#{counter := Num} = State) -> +counter_loop() -> + counter_loop(#{counter => 0, status => running}). + +counter_loop(#{counter := Num, status := Status} = State) -> NewState = receive - {inc, N} -> - #{counter => Num + N}; + block -> + ct:pal("counter recv: ~p", [block]), + State#{status => blocked}; + resume -> + {messages, Msgs} = erlang:process_info(self(), messages), + ct:pal("counter recv: ~p, buffered msgs: ~p", [resume, length(Msgs)]), + State#{status => running}; + {inc, N, ReplyFun} when Status == running -> + apply_reply(ReplyFun, ok), + State#{counter => Num + N}; + {{FromPid, ReqRef}, {inc, N}} when Status == running -> + FromPid ! {ReqRef, ok}, + State#{counter => Num + N}; + {{FromPid, ReqRef}, {inc, _N}} when Status == blocked -> + FromPid ! {ReqRef, incorrect_status}, + State; + {get, ReplyFun} -> + apply_reply(ReplyFun, Num), + State; {{FromPid, ReqRef}, get} -> FromPid ! {ReqRef, Num}, State diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 5177e792c..5c62e22c2 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -236,7 +236,7 @@ t_batch_query_counter(_) -> ok = emqx_resource:remove_local(?ID). -t_query_counter_async(_) -> +t_query_counter_async_query(_) -> {ok, _} = emqx_resource:create_local( ?ID, ?DEFAULT_RESOURCE_GROUP, @@ -271,24 +271,25 @@ t_query_counter_async(_) -> ?assertMatch(#{matched := 1002, success := 1002, failed := 0}, C), ok = emqx_resource:remove_local(?ID). -t_query_counter_async_2(_) -> +t_query_counter_async_callback(_) -> emqx_connector_demo:set_callback_mode(async_if_possible), Tab0 = ets:new(?FUNCTION_NAME, [bag, public]), Insert = fun(Tab, Result) -> ets:insert(Tab, {make_ref(), Result}) end, + ReqOpts = #{async_reply_fun => {Insert, [Tab0]}}, {ok, _} = emqx_resource:create_local( ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{query_mode => async, async_reply_fun => {Insert, [Tab0]}} + #{query_mode => async, async_inflight_window => 1000000} ), ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), ?check_trace( ?TRACE_OPTS, - inc_counter_in_parallel(1000), + inc_counter_in_parallel(1000, ReqOpts), fun(Trace) -> QueryTrace = ?of_kind(call_query_async, Trace), ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace) @@ -321,6 +322,117 @@ t_query_counter_async_2(_) -> ), ok = emqx_resource:remove_local(?ID). +t_query_counter_async_inflight(_) -> + emqx_connector_demo:set_callback_mode(async_if_possible), + + Tab0 = ets:new(?FUNCTION_NAME, [bag, public]), + Insert0 = fun(Tab, Result) -> + ets:insert(Tab, {make_ref(), Result}) + end, + ReqOpts = #{async_reply_fun => {Insert0, [Tab0]}}, + WindowSize = 15, + {ok, _} = emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, register => true}, + #{ + query_mode => async, + async_inflight_window => WindowSize, + worker_pool_size => 1, + resume_interval => 300 + } + ), + ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), + + %% block the resource + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)), + + %% send async query to make the inflight window full + ?check_trace( + ?TRACE_OPTS, + inc_counter_in_parallel(WindowSize, ReqOpts), + fun(Trace) -> + QueryTrace = ?of_kind(call_query_async, Trace), + ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace) + end + ), + + %% this will block the resource_worker + ok = emqx_resource:query(?ID, {inc_counter, 1}), + ?assertMatch(0, ets:info(Tab0, size)), + %% sleep to make the resource_worker resume some times + timer:sleep(2000), + + %% send query now will fail because the resource is blocked. + Insert = fun(Tab, Ref, Result) -> + ets:insert(Tab, {Ref, Result}) + end, + ok = emqx_resource:query(?ID, {inc_counter, 1}, #{ + async_reply_fun => {Insert, [Tab0, tmp_query]} + }), + ?assertMatch([{_, {error, {resource_error, #{reason := blocked}}}}], ets:take(Tab0, tmp_query)), + + %% all response should be received after the resource is resumed. + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)), + timer:sleep(1000), + ?assertEqual(WindowSize, ets:info(Tab0, size)), + + %% send async query, this time everything should be ok. + Num = 10, + ?check_trace( + ?TRACE_OPTS, + inc_counter_in_parallel(Num, ReqOpts), + fun(Trace) -> + QueryTrace = ?of_kind(call_query_async, Trace), + ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace) + end + ), + timer:sleep(1000), + ?assertEqual(WindowSize + Num, ets:info(Tab0, size)), + + %% block the resource + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, block)), + %% again, send async query to make the inflight window full + ?check_trace( + ?TRACE_OPTS, + inc_counter_in_parallel(WindowSize, ReqOpts), + fun(Trace) -> + QueryTrace = ?of_kind(call_query_async, Trace), + ?assertMatch([#{query := {query, _, {inc_counter, 1}}} | _], QueryTrace) + end + ), + + %% this will block the resource_worker + ok = emqx_resource:query(?ID, {inc_counter, 1}), + + Sent = WindowSize + Num + WindowSize, + ?assertMatch(ok, emqx_resource:simple_sync_query(?ID, resume)), + timer:sleep(1000), + ?assertEqual(Sent, ets:info(Tab0, size)), + + {ok, Counter} = emqx_resource:simple_sync_query(?ID, get_counter), + ct:pal("get_counter: ~p, sent: ~p", [Counter, Sent]), + ?assert(Sent == Counter), + + {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), + ct:pal("metrics: ~p", [C]), + ?assertMatch( + #{matched := M, success := S, exception := E, failed := F, resource_down := RD} when + M >= Sent andalso M == S + E + F + RD, + C + ), + ?assert( + lists:all( + fun + ({_, ok}) -> true; + (_) -> false + end, + ets:tab2list(Tab0) + ) + ), + ok = emqx_resource:remove_local(?ID). + t_healthy_timeout(_) -> {ok, _} = emqx_resource:create_local( ?ID, @@ -550,10 +662,13 @@ t_auto_retry(_) -> %% Helpers %%------------------------------------------------------------------------------ inc_counter_in_parallel(N) -> + inc_counter_in_parallel(N, #{}). + +inc_counter_in_parallel(N, Opts) -> Parent = self(), Pids = [ erlang:spawn(fun() -> - emqx_resource:query(?ID, {inc_counter, 1}), + emqx_resource:query(?ID, {inc_counter, 1}, Opts), Parent ! {complete, self()} end) || _ <- lists:seq(1, N) @@ -567,6 +682,17 @@ inc_counter_in_parallel(N) -> || Pid <- Pids ]. +% verify_inflight_full(WindowSize) -> +% ?check_trace( +% ?TRACE_OPTS, +% emqx_resource:query(?ID, {inc_counter, 1}), +% fun(Return, Trace) -> +% QueryTrace = ?of_kind(inflight_full, Trace), +% ?assertMatch([#{wind_size := WindowSize} | _], QueryTrace), +% ?assertMatch(ok, Return) +% end +% ). + bin_config() -> <<"\"name\": \"test_resource\"">>. From 0090a3ee9393f00ecfd6285dab79979a206eacda Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 10 Aug 2022 11:47:14 +0800 Subject: [PATCH 44/71] fix(influxdb): fix illegal `wirte_syntax` example --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 83c5a4127..53d02fa6d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -51,7 +51,7 @@ values(Protocol, get) -> values(Protocol, post) -> case Protocol of "influxdb_api_v2" -> - SupportUint = <<"uint_value=${payload.uint_key}u">>; + SupportUint = <<"uint_value=${payload.uint_key}u,">>; _ -> SupportUint = <<>> end, From 223b84017edd0d557498a184678c15b3590a6340 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 11:35:04 +0800 Subject: [PATCH 45/71] fix(influxdb): api schema `write_syntax` using raw type `string()` --- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 7 +++++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 11 ++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 59b320368..68514fdb0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -656,6 +656,13 @@ typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; +typename_to_spec("write_syntax()", _Mod) -> + #{ + type => string, + example => + <<"${topic},clientid=${clientid}", " ", "payload=${payload},", + "${clientid}_int_value=${payload.int_key}i,", "bool=${payload.bool}">> + }; typename_to_spec("ip_ports()", _Mod) -> #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>}; typename_to_spec("url()", _Mod) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 53d02fa6d..7de640040 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -21,6 +21,11 @@ desc/1 ]). +-type write_syntax() :: list(). +-reflect_type([write_syntax/0]). +-typerefl_from_string({write_syntax/0, ?MODULE, to_influx_lines}). +-export([to_influx_lines/1]). + %% ------------------------------------------------------------------------------------------------- %% api @@ -148,19 +153,19 @@ desc(_) -> undefined. write_syntax(type) -> - list(); + ?MODULE:write_syntax(); write_syntax(required) -> true; write_syntax(validator) -> [?NOT_EMPTY("the value of the field 'write_syntax' cannot be empty")]; write_syntax(converter) -> - fun converter_influx_lines/1; + fun to_influx_lines/1; write_syntax(desc) -> ?DESC("write_syntax"); write_syntax(_) -> undefined. -converter_influx_lines(RawLines) -> +to_influx_lines(RawLines) -> Lines = string:tokens(str(RawLines), "\n"), lists:reverse(lists:foldl(fun converter_influx_line/2, [], Lines)). From 7581082fcbd86b6090cc93a2b9345b6bcb52cd30 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 11 Aug 2022 13:22:16 +0800 Subject: [PATCH 46/71] fix(bridge): replace prepare_statement by sql_template --- .../src/emqx_connector_mysql.erl | 16 ++++++++++++++-- .../i18n/emqx_ee_bridge_mysql.conf | 2 +- .../src/emqx_ee_bridge_mysql.erl | 18 ++++-------------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index b379e511c..cae07433a 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -111,7 +111,7 @@ on_start( {pool_size, PoolSize} ], PoolName = emqx_plugin_libs_pool:pool_name(InstId), - Prepares = parse_prepare_sql(maps:get(prepare_statement, Config, #{})), + Prepares = parse_prepare_sql(Config), State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of ok -> {ok, init_prepare(State)}; @@ -303,7 +303,19 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> unprepare_sql_to_conn(Conn, PrepareSqlKey) -> mysql:unprepare(Conn, PrepareSqlKey). -parse_prepare_sql(SQL) -> +parse_prepare_sql(Config) -> + SQL = + case maps:get(prepare_statement, Config, undefined) of + undefined -> + case emqx_map_lib:deep_get([egress, sql_template], Config, undefined) of + undefined -> + #{}; + Template -> + #{send_message => Template} + end; + Any -> + Any + end, parse_prepare_sql(maps:to_list(SQL), #{}, #{}). parse_prepare_sql([{Key, H} | T], SQL, Tokens) -> diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf index 9e3796487..48fbd1007 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf @@ -1,5 +1,5 @@ emqx_ee_bridge_mysql { - sql { + sql_template { desc { en: """SQL Template""" zh: """SQL 模板""" diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index bf38da2f8..ebcbc0e64 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -44,14 +44,14 @@ values(post) -> #{ type => mysql, name => <<"mysql">>, + sql_template => ?DEFAULT_SQL, connector => #{ server => <<"127.0.0.1:3306">>, database => <<"test">>, pool_size => 8, username => <<"root">>, password => <<"public">>, - auto_reconnect => true, - prepare_statement => #{send_message => ?DEFAULT_SQL} + auto_reconnect => true }, enable => true, direction => egress @@ -69,6 +69,7 @@ fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, + {sql_template, mk(binary(), #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL})}, {connector, mk( ref(?MODULE, connector), @@ -85,8 +86,7 @@ fields("put") -> fields("get") -> emqx_bridge_schema:metrics_status_fields() ++ fields("post"); fields(connector) -> - (emqx_connector_mysql:fields(config) -- - emqx_connector_schema_lib:prepare_statement_fields()) ++ prepare_statement_fields(). + emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields(). desc("config") -> ?DESC("desc_config"); @@ -104,13 +104,3 @@ type_field() -> name_field() -> {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. - -prepare_statement_fields() -> - [ - {prepare_statement, - mk(map(), #{ - desc => ?DESC(emqx_connector_schema_lib, prepare_statement), - default => #{<<"send_message">> => ?DEFAULT_SQL}, - example => #{<<"send_message">> => ?DEFAULT_SQL} - })} - ]. From dc804993999de0b46175d00e59b3521c016f45e5 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 11 Aug 2022 15:58:48 +0800 Subject: [PATCH 47/71] fix(bridge): add sql_template field format --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index ebcbc0e64..5d143bf85 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -69,7 +69,11 @@ fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, - {sql_template, mk(binary(), #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL})}, + {sql_template, + mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )}, {connector, mk( ref(?MODULE, connector), From 22a4ca311c9e9320260d90d9d2eb2ecc2c1d6cd5 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 15:09:05 +0800 Subject: [PATCH 48/71] feat(resource): resource batch/async/queue config schema --- .../i18n/emqx_resource_schema_i18n.conf | 119 ++++++++++++++++++ apps/emqx_resource/include/emqx_resource.hrl | 12 ++ .../src/emqx_resource_worker.erl | 17 +-- .../src/schema/emqx_resource_schema.erl | 99 +++++++++++++++ .../test/emqx_resource_SUITE.erl | 2 +- 5 files changed, 235 insertions(+), 14 deletions(-) create mode 100644 apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf create mode 100644 apps/emqx_resource/src/schema/emqx_resource_schema.erl diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf new file mode 100644 index 000000000..a3fb6c402 --- /dev/null +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -0,0 +1,119 @@ +emqx_resource_schema { + batch { + desc { + en: """ +Configuration of batch query.
+Batch requests are made immediately when the number of requests reaches the `batch_size`, or also immediately when the number of requests is less than the batch request size but the maximum batch_time has been reached. +""" + zh: """ +批量请求配置。
+请求数达到批量请求大小时立刻进行批量请求，或当请求数不足批量请求数大小，但已经达到最大批量等待时间时也立即进行批量请求。 +""" + } + label { + en: """batch""" + zh: """批量请求""" + } + } + + queue { + desc { + en: """Configuration of queue.""" + zh: """请求队列配置""" + } + label { + en: """queue""" + zh: """请求队列""" + } + } + + query_mode { + desc { + en: """Query mode. Optional 'sync/async', default 'sync'.""" + zh: """请求模式。可选 '同步/异步'，默认为'同步'模式。""" + } + label { + en: """query_mode""" + zh: """query_mode""" + } + } + + enable_batch { + desc { + en: """Batch mode enabled.""" + zh: """启用批量模式。""" + } + label { + en: """enable_batch""" + zh: """启用批量模式""" + } + } + + enable_queue { + desc { + en: """Queue mode enabled.""" + zh: """启用队列模式。""" + } + label { + en: """enable_queue""" + zh: """启用队列模式""" + } + } + + resume_interval { + desc { + en: """Resume time interval when resource down.""" + zh: """资源不可用时的重试时间""" + } + label { + en: """resume_interval""" + zh: """恢复时间""" + } + } + + async_inflight_window { + desc { + en: """Async queyr inflight window.""" + zh: """异步请求飞行队列窗口大小""" + } + label { + en: """async_inflight_window""" + zh: """异步请求飞行队列窗口""" + } + } + + batch_size { + desc { + en: """Maximum batch count.""" + zh: """批量请求大小""" + } + label { + en: """batch_size""" + zh: """批量请求大小""" + } + } + + batch_time { + desc { + en: """Maximum batch waiting interval.""" + zh: """最大批量请求等待时间。""" + } + label { + en: """batch_time""" + zh: """批量等待间隔""" + } + } + + queue_max_bytes { + desc { + en: """Maximum queue storage size in bytes.""" + zh: """消息队列的最大长度，以字节计。""" + } + label { + en: """queue_max_bytes""" + zh: """队列最大长度""" + } + } + + +} diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 5c561a8d3..5327e3aae 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -60,5 +60,17 @@ | {error, term()} | {resource_down, term()}. +%% count +-define(DEFAULT_BATCH_SIZE, 100). +%% milliseconds +-define(DEFAULT_BATCH_TIME, 10). + +%% bytes +-define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024). + +-define(DEFAULT_INFLIGHT, 100). + +-define(RESUME_INTERVAL, 15000). + -define(TEST_ID_PREFIX, "_test_:"). -define(RES_METRICS, resource_metrics). diff --git a/apps/emqx_resource/src/emqx_resource_worker.erl b/apps/emqx_resource/src/emqx_resource_worker.erl index 716842bf9..e940dcb69 100644 --- a/apps/emqx_resource/src/emqx_resource_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_worker.erl @@ -52,13 +52,6 @@ -export([reply_after_query/6, batch_reply_after_query/6]). --define(RESUME_INTERVAL, 15000). - -%% count --define(DEFAULT_BATCH_SIZE, 100). -%% milliseconds --define(DEFAULT_BATCH_TIME, 10). - -define(Q_ITEM(REQUEST), {q_item, REQUEST}). -define(QUERY(FROM, REQUEST), {query, FROM, REQUEST}). @@ -69,8 +62,6 @@ {error, {resource_error, #{reason => Reason, msg => iolist_to_binary(Msg)}}} ). -define(RESOURCE_ERROR_M(Reason, Msg), {error, {resource_error, #{reason := Reason, msg := Msg}}}). --define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024). --define(DEFAULT_INFLIGHT, 100). -type id() :: binary(). -type query() :: {query, from(), request()}. @@ -122,7 +113,7 @@ init({Id, Index, Opts}) -> Name = name(Id, Index), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), Queue = - case maps:get(queue_enabled, Opts, false) of + case maps:get(enable_queue, Opts, false) of true -> replayq:open(#{ dir => disk_queue_dir(Id, Index), @@ -144,7 +135,7 @@ init({Id, Index, Opts}) -> %% if the resource worker is overloaded query_mode => maps:get(query_mode, Opts, sync), async_inflight_window => maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT), - batch_enabled => maps:get(batch_enabled, Opts, false), + enable_batch => maps:get(enable_batch, Opts, false), batch_size => BatchSize, batch_time => maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), queue => Queue, @@ -270,14 +261,14 @@ drop_head(Q) -> ok = replayq:ack(Q1, AckRef), Q1. -query_or_acc(From, Request, #{batch_enabled := true, acc := Acc, acc_left := Left} = St0) -> +query_or_acc(From, Request, #{enable_batch := true, acc := Acc, acc_left := Left} = St0) -> Acc1 = [?QUERY(From, Request) | Acc], St = St0#{acc := Acc1, acc_left := Left - 1}, case Left =< 1 of true -> flush(St); false -> {keep_state, ensure_flush_timer(St)} end; -query_or_acc(From, Request, #{batch_enabled := false, queue := Q, id := Id, query_mode := QM} = St) -> +query_or_acc(From, Request, #{enable_batch := false, queue := Q, id := Id, query_mode := QM} = St) -> QueryOpts = #{ inflight_name => maps:get(name, St), inflight_window => maps:get(async_inflight_window, St) diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl new file mode 100644 index 000000000..933cd0189 --- /dev/null +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_schema). + +-include("emqx_resource.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([namespace/0, roots/0, fields/1]). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions + +namespace() -> "resource_schema". + +roots() -> []. + +fields('batch&async&queue') -> + [ + {query_mode, fun query_mode/1}, + {resume_interval, fun resume_interval/1}, + {async_inflight_window, fun async_inflight_window/1}, + {batch, mk(ref(?MODULE, batch), #{desc => ?DESC("batch")})}, + {queue, mk(ref(?MODULE, queue), #{desc => ?DESC("queue")})} + ]; +fields(batch) -> + [ + {enable_batch, fun enable_batch/1}, + {batch_size, fun batch_size/1}, + {batch_time, fun batch_time/1} + ]; +fields(queue) -> + [ + {enable_queue, fun enable_queue/1}, + {max_queue_bytes, fun queue_max_bytes/1} + ]. + +query_mode(type) -> enum([sync, async]); +query_mode(desc) -> ?DESC("query_mode"); +query_mode(default) -> sync; +query_mode(required) -> false; +query_mode(_) -> undefined. + +enable_batch(type) -> boolean(); +enable_batch(required) -> false; +enable_batch(default) -> false; +enable_batch(desc) -> ?DESC("enable_batch"); +enable_batch(_) -> undefined. + +enable_queue(type) -> boolean(); +enable_queue(required) -> false; +enable_queue(default) -> false; +enable_queue(desc) -> ?DESC("enable_queue"); +enable_queue(_) -> undefined. + +resume_interval(type) -> emqx_schema:duration_ms(); +resume_interval(desc) -> ?DESC("resume_interval"); +resume_interval(default) -> ?RESUME_INTERVAL; +resume_interval(required) -> false; +resume_interval(_) -> undefined. + +async_inflight_window(type) -> pos_integer(); +async_inflight_window(desc) -> ?DESC("async_inflight_window"); +async_inflight_window(default) -> ?DEFAULT_INFLIGHT; +async_inflight_window(required) -> false; +async_inflight_window(_) -> undefined. + +batch_size(type) -> pos_integer(); +batch_size(desc) -> ?DESC("batch_size"); +batch_size(default) -> ?DEFAULT_BATCH_SIZE; +batch_size(required) -> false; +batch_size(_) -> undefined. + +batch_time(type) -> emqx_schema:duration_ms(); +batch_time(desc) -> ?DESC("batch_time"); +batch_time(default) -> ?DEFAULT_BATCH_TIME; +batch_time(required) -> false; +batch_time(_) -> undefined. + +queue_max_bytes(type) -> emqx_schema:bytesize(); +queue_max_bytes(desc) -> ?DESC("queue_max_bytes"); +queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE; +queue_max_bytes(required) -> false; +queue_max_bytes(_) -> undefined. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 5c62e22c2..ddd671b75 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -211,7 +211,7 @@ t_batch_query_counter(_) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{batch_enabled => true} + #{enable_batch => true} ), ?check_trace( From 2843d33f7a33fa46bb12ae09b788944b99870403 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 15:10:25 +0800 Subject: [PATCH 49/71] chore: refine md form format --- deploy/charts/emqx/README.md | 108 +++++++++++++++++------------------ 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 496d52061..acbbcf95e 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -1,5 +1,5 @@ # Introduction -This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. +This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. # Prerequisites + Kubernetes 1.6+ @@ -8,7 +8,7 @@ This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm # Installing the Chart To install the chart with the release name `my-emqx`: -+ From github ++ From github ``` $ git clone https://github.com/emqx/emqx.git $ cd emqx/deploy/charts/emqx @@ -31,58 +31,58 @@ $ helm del my-emqx # Configuration The following table lists the configurable parameters of the emqx chart and their default values. -| Parameter | Description | Default Value | -| --- | --- | --- | -| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. |3| -| `image.repository` | EMQX Image name |emqx/emqx| -| `image.pullPolicy` | The image pull policy |IfNotPresent| -| `image.pullSecrets ` | The image pull secrets |`[]` (does not add image pull secrets to deployed pods)| -| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | -| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | -`podAnnotations ` | Annotations for pod | `{}` -`podManagementPolicy`| To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel` -| `persistence.enabled` | Enable EMQX persistence using PVC |false| -| `persistence.storageClass` | Storage class of backing PVC |`nil` (uses alpha storage class annotation)| -| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template |""| -| `persistence.accessMode` | PVC Access Mode for EMQX volume |ReadWriteOnce| -| `persistence.size` | PVC Storage Request for EMQX volume |20Mi| -| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`| -| `resources` | CPU/Memory resource requests/limits |{}| -| `nodeSelector` | Node labels for pod assignment |`{}`| -| `tolerations` | Toleration labels for pod assignment |`[]`| -| `affinity` | Map of node/pod affinities |`{}`| -| `service.type` | Kubernetes Service type. |ClusterIP| -| `service.mqtt` | Port for MQTT. |1883| -| `service.mqttssl` | Port for MQTT(SSL). |8883| -| `service.mgmt` | Port for mgmt API. |8081| -| `service.ws` | Port for WebSocket/HTTP. |8083| -| `service.wss` | Port for WSS/HTTPS. |8084| -| `service.dashboard` | Port for dashboard. |18083| -| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. |nil| -| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). |nil| -| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. |nil| -| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. |nil| -| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. |nil| -| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. |nil| -| `service.loadBalancerIP` | loadBalancerIP for Service | nil | -| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | -| `service.externalIPs` | ExternalIPs for the service | [] | -| `service.annotations` | Service annotations | {}(evaluated as a template)| -| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | -| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | -| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` -| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | -| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | -| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | -| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | -| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | -| `metrics.type` | Now we only supported "prometheus" | "prometheus" | +| Parameter | Description | Default Value | +|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 | +| `image.repository` | EMQX Image name | emqx/emqx | +| `image.pullPolicy` | The image pull policy | IfNotPresent | +| `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | +| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | +| `podAnnotations ` | Annotations for pod | `{}` | +| `podManagementPolicy` | To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel` | +| `persistence.enabled` | Enable EMQX persistence using PVC | false | +| `persistence.storageClass` | Storage class of backing PVC | `nil` (uses alpha storage class annotation) | +| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template | "" | +| `persistence.accessMode` | PVC Access Mode for EMQX volume | ReadWriteOnce | +| `persistence.size` | PVC Storage Request for EMQX volume | 20Mi | +| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. | `{}` | +| `resources` | CPU/Memory resource requests/limits | {} | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | +| `service.type` | Kubernetes Service type. | ClusterIP | +| `service.mqtt` | Port for MQTT. | 1883 | +| `service.mqttssl` | Port for MQTT(SSL). | 8883 | +| `service.mgmt` | Port for mgmt API. | 8081 | +| `service.ws` | Port for WebSocket/HTTP. | 8083 | +| `service.wss` | Port for WSS/HTTPS. | 8084 | +| `service.dashboard` | Port for dashboard. | 18083 | +| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | +| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | +| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | +| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | +| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | +| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | +| `service.loadBalancerIP` | loadBalancerIP for Service | nil | +| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | +| `service.externalIPs` | ExternalIPs for the service | [] | +| `service.annotations` | Service annotations | {}(evaluated as a template) | +| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | +| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | +| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | +| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | +| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | +| `metrics.type` | Now we only supported "prometheus" | "prometheus" | ## EMQX specific settings The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their default values. From 0f6c3717602e20f77635831ea25eaf90b3d2ff7e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 18:12:41 +0800 Subject: [PATCH 50/71] feat(influxdb): influxdb connector add `on_batch_query/3` callback --- apps/emqx_resource/src/emqx_resource.erl | 1 + .../src/emqx_ee_bridge_influxdb.erl | 9 +- .../src/emqx_ee_connector_influxdb.erl | 106 ++++++++++++++---- 3 files changed, 91 insertions(+), 25 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 0d2289696..6601b9eea 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -126,6 +126,7 @@ %% when calling emqx_resource:query/3 -callback on_query(resource_id(), Request :: term(), resource_state()) -> query_result(). +%% when calling emqx_resource:on_batch_query/3 -callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result(). %% when calling emqx_resource:health_check/2 diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 7de640040..4edeb786a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -70,7 +70,8 @@ values(Protocol, post) -> write_syntax => <<"${topic},clientid=${clientid}", " ", "payload=${payload},", "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary, - "bool=${payload.bool}">> + "bool=${payload.bool}">>, + batch => #{enable_batch => false, batch_size => 5, batch_time => <<"1m">>} }; values(Protocol, put) -> values(Protocol, post). @@ -109,7 +110,9 @@ fields("get_api_v2") -> fields(Name) when Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2 -> - fields(basic) ++ connector_field(Name). + fields(basic) ++ + emqx_resource_schema:fields('batch&async&queue') ++ + connector_field(Name). method_fileds(post, ConnectorType) -> fields(basic) ++ connector_field(ConnectorType) ++ type_name_field(ConnectorType); @@ -162,6 +165,8 @@ write_syntax(converter) -> fun to_influx_lines/1; write_syntax(desc) -> ?DESC("write_syntax"); +write_syntax(format) -> + <<"sql">>; write_syntax(_) -> undefined. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 09b3d7350..41f7059ec 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -17,6 +17,7 @@ on_start/2, on_stop/2, on_query/3, + on_batch_query/3, on_get_status/2 ]). @@ -37,8 +38,29 @@ on_start(InstId, Config) -> on_stop(_InstId, #{client := Client}) -> influxdb:stop_client(Client). -on_query(InstId, {send_message, Data}, State) -> - do_query(InstId, {send_message, Data}, State). +on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, client := Client}) -> + case data_to_points(Data, SyntaxLines) of + {ok, Points} -> + do_query(InstId, Client, Points); + {error, ErrorPoints} = Err -> + log_error_points(InstId, ErrorPoints), + Err + end. + +%% Once a Batched Data trans to points failed. +%% This batch query failed +on_batch_query(InstId, BatchData, State = #{write_syntax := SyntaxLines, client := Client}) -> + case on_get_status(InstId, State) of + connected -> + case parse_batch_data(InstId, BatchData, SyntaxLines) of + {ok, Points} -> + do_query(InstId, Client, Points); + {error, Reason} -> + {error, Reason} + end; + disconnected -> + {resource_down, disconnected} + end. on_get_status(_InstId, #{client := Client}) -> case influxdb:is_alive(Client) of @@ -79,7 +101,7 @@ fields("api_v2_put") -> fields(basic) -> [ {host, - mk(binary(), #{required => true, default => <<"120.0.0.1">>, desc => ?DESC("host")})}, + mk(binary(), #{required => true, default => <<"127.0.0.1">>, desc => ?DESC("host")})}, {port, mk(pos_integer(), #{required => true, default => 8086, desc => ?DESC("port")})}, {precision, mk(enum([ns, us, ms, s, m, h]), #{ @@ -310,18 +332,7 @@ ssl_config(SSL = #{enable := true}) -> %% ------------------------------------------------------------------------------------------------- %% Query -do_query(InstId, {send_message, Data}, State = #{client := Client}) -> - {Points, Errs} = data_to_points(Data, State), - lists:foreach( - fun({error, Reason}) -> - ?SLOG(error, #{ - msg => "influxdb trans point failed", - connector => InstId, - reason => Reason - }) - end, - Errs - ), +do_query(InstId, Client, Points) -> case influxdb:write(Client, Points) of ok -> ?SLOG(debug, #{ @@ -376,11 +387,45 @@ to_maps_config(K, V, Res) -> %% ------------------------------------------------------------------------------------------------- %% Tags & Fields Data Trans -data_to_points(Data, #{write_syntax := Lines}) -> - lines_to_points(Data, Lines, [], []). +parse_batch_data(InstId, BatchData, SyntaxLines) -> + {Points, Errors} = lists:foldl( + fun({send_message, Data}, {AccIn, ErrAccIn}) -> + case data_to_points(Data, SyntaxLines) of + {ok, Points} -> + {[Points | AccIn], ErrAccIn}; + {error, ErrorPoints} -> + log_error_points(InstId, ErrorPoints), + {AccIn, ErrAccIn + 1} + end + end, + {[], 0}, + BatchData + ), + case Errors of + 0 -> + {ok, Points}; + _ -> + ?SLOG(error, #{ + msg => io_lib:format("InfluxDB trans point failed, count: ~p", [Errors]), + connector => InstId, + reason => points_trans_failed + }), + {error, points_trans_failed} + end. -lines_to_points(_, [], Points, Errs) -> - {Points, Errs}; +data_to_points(Data, SyntaxLines) -> + lines_to_points(Data, SyntaxLines, [], []). + +%% When converting multiple rows data into InfluxDB Line Protocol, they are considered to be strongly correlated. +%% And once a row fails to convert, all of them are considered to have failed. +lines_to_points(_, [], Points, ErrorPoints) -> + case ErrorPoints of + [] -> + {ok, Points}; + _ -> + %% ignore trans succeeded points + {error, ErrorPoints} + end; lines_to_points( Data, [ @@ -392,8 +437,8 @@ lines_to_points( } | Rest ], - ResAcc, - ErrAcc + ResultPointsAcc, + ErrorPointsAcc ) -> TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of @@ -406,9 +451,11 @@ lines_to_points( tags => EncodeTags, fields => EncodeFields }, - lines_to_points(Data, Rest, [Point | ResAcc], ErrAcc); + lines_to_points(Data, Rest, [Point | ResultPointsAcc], ErrorPointsAcc); BadTimestamp -> - lines_to_points(Data, Rest, ResAcc, [{error, {bad_timestamp, BadTimestamp}} | ErrAcc]) + lines_to_points(Data, Rest, ResultPointsAcc, [ + {error, {bad_timestamp, BadTimestamp}} | ErrorPointsAcc + ]) end. maps_config_to_data(K, V, {Data, Res}) -> @@ -461,3 +508,16 @@ data_filter(Bool) when is_boolean(Bool) -> Bool; data_filter(Data) -> bin(Data). bin(Data) -> emqx_plugin_libs_rule:bin(Data). + +%% helper funcs +log_error_points(InstId, Errs) -> + lists:foreach( + fun({error, Reason}) -> + ?SLOG(error, #{ + msg => "influxdb trans point failed", + connector => InstId, + reason => Reason + }) + end, + Errs + ). From 2872f0b6685061fa4242d829b6f33cdf88e5f76c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 11 Aug 2022 19:11:44 +0800 Subject: [PATCH 51/71] fix(bridges): support create resources with options --- apps/emqx_bridge/src/emqx_bridge.erl | 9 +++-- apps/emqx_bridge/src/emqx_bridge_resource.erl | 11 +++-- .../i18n/emqx_resource_schema_i18n.conf | 28 ------------- apps/emqx_resource/include/emqx_resource.hrl | 12 +++++- apps/emqx_resource/src/emqx_resource.erl | 40 ++++++++++++++----- .../src/emqx_resource_manager.erl | 8 ++-- .../src/proto/emqx_resource_proto_v1.erl | 4 +- .../src/schema/emqx_resource_schema.erl | 12 +----- .../src/emqx_ee_bridge_influxdb.erl | 2 +- 9 files changed, 61 insertions(+), 65 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index ba6c64dbc..c794a25a5 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -171,9 +171,9 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnv) -> diff_confs(NewConf, OldConf), %% The config update will be failed if any task in `perform_bridge_changes` failed. Result = perform_bridge_changes([ - {fun emqx_bridge_resource:remove/3, Removed}, - {fun emqx_bridge_resource:create/3, Added}, - {fun emqx_bridge_resource:update/3, Updated} + {fun emqx_bridge_resource:remove/4, Removed}, + {fun emqx_bridge_resource:create/4, Added}, + {fun emqx_bridge_resource:update/4, Updated} ]), ok = unload_hook(), ok = load_hook(NewConf), @@ -261,7 +261,8 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> ({_Type, _Name}, _Conf, {error, Reason}) -> {error, Reason}; ({Type, Name}, Conf, _) -> - case Action(Type, Name, Conf) of + ResOpts = emqx_resource:fetch_creation_opts(Conf), + case Action(Type, Name, Conf, ResOpts) of {error, Reason} -> {error, Reason}; Return -> Return end diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 35ace560c..ac1ec6ba3 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -158,11 +158,14 @@ recreate(Type, Name) -> recreate(Type, Name, emqx:get_config([bridges, Type, Name])). recreate(Type, Name, Conf) -> + recreate(Type, Name, Conf, #{}). + +recreate(Type, Name, Conf, Opts) -> emqx_resource:recreate_local( resource_id(Type, Name), bridge_to_resource_type(Type), parse_confs(Type, Name, Conf), - #{auto_retry_interval => 60000} + Opts#{auto_retry_interval => 60000} ). create_dry_run(Type, Conf) -> @@ -186,13 +189,13 @@ create_dry_run(Type, Conf) -> remove(BridgeId) -> {BridgeType, BridgeName} = parse_bridge_id(BridgeId), - remove(BridgeType, BridgeName, #{}). + remove(BridgeType, BridgeName, #{}, #{}). remove(Type, Name) -> - remove(Type, Name, undefined). + remove(Type, Name, #{}, #{}). %% just for perform_bridge_changes/1 -remove(Type, Name, _Conf) -> +remove(Type, Name, _Conf, _Opts) -> ?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}), case emqx_resource:remove_local(resource_id(Type, Name)) of ok -> ok; diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index a3fb6c402..abdb220bb 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -1,32 +1,4 @@ emqx_resource_schema { - batch { - desc { - en: """ -Configuration of batch query.
-Batch requests are made immediately when the number of requests reaches the `batch_size`, or also immediately when the number of requests is less than the batch request size but the maximum batch_time has been reached. -""" - zh: """ -批量请求配置。
-请求数达到批量请求大小时立刻进行批量请求，或当请求数不足批量请求数大小，但已经达到最大批量等待时间时也立即进行批量请求。 -""" - } - label { - en: """batch""" - zh: """批量请求""" - } - } - - queue { - desc { - en: """Configuration of queue.""" - zh: """请求队列配置""" - } - label { - en: """queue""" - zh: """请求队列""" - } - } - query_mode { desc { en: """Query mode. Optional 'sync/async', default 'sync'.""" diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 5327e3aae..5b6856dc0 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -40,7 +40,7 @@ metrics := emqx_metrics_worker:metrics() }. -type resource_group() :: binary(). --type create_opts() :: #{ +-type creation_opts() :: #{ health_check_interval => integer(), health_check_timeout => integer(), %% We can choose to block the return of emqx_resource:start until @@ -52,7 +52,15 @@ start_after_created => boolean(), %% If the resource disconnected, we can set to retry starting the resource %% periodically. - auto_retry_interval => integer() + auto_retry_interval => integer(), + enable_batch => boolean(), + batch_size => integer(), + batch_time => integer(), + enable_queue => boolean(), + queue_max_bytes => integer(), + query_mode => async | sync | dynamic + resume_interval => integer(), + async_inflight_window => integer() }. -type query_result() :: ok diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 6601b9eea..b134d2af1 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -102,6 +102,7 @@ list_instances_verbose/0, %% return the data of the instance get_instance/1, + fetch_creation_opts/1, %% return all the instances of the same resource type list_instances_by_type/1, generate_id/1, @@ -159,7 +160,7 @@ is_resource_mod(Module) -> create(ResId, Group, ResourceType, Config) -> create(ResId, Group, ResourceType, Config, #{}). --spec create(resource_id(), resource_group(), resource_type(), resource_config(), create_opts()) -> +-spec create(resource_id(), resource_group(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. create(ResId, Group, ResourceType, Config, Opts) -> emqx_resource_proto_v1:create(ResId, Group, ResourceType, Config, Opts). @@ -175,7 +176,7 @@ create_local(ResId, Group, ResourceType, Config) -> resource_group(), resource_type(), resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data()}. create_local(ResId, Group, ResourceType, Config, Opts) -> @@ -196,7 +197,7 @@ create_dry_run_local(ResourceType, Config) -> recreate(ResId, ResourceType, Config) -> recreate(ResId, ResourceType, Config, #{}). --spec recreate(resource_id(), resource_type(), resource_config(), create_opts()) -> +-spec recreate(resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. recreate(ResId, ResourceType, Config, Opts) -> emqx_resource_proto_v1:recreate(ResId, ResourceType, Config, Opts). @@ -206,7 +207,7 @@ recreate(ResId, ResourceType, Config, Opts) -> recreate_local(ResId, ResourceType, Config) -> recreate_local(ResId, ResourceType, Config, #{}). --spec recreate_local(resource_id(), resource_type(), resource_config(), create_opts()) -> +-spec recreate_local(resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. recreate_local(ResId, ResourceType, Config, Opts) -> emqx_resource_manager:recreate(ResId, ResourceType, Config, Opts). @@ -249,7 +250,7 @@ simple_async_query(ResId, Request, ReplyFun) -> start(ResId) -> start(ResId, #{}). --spec start(resource_id(), create_opts()) -> ok | {error, Reason :: term()}. +-spec start(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}. start(ResId, Opts) -> emqx_resource_manager:start(ResId, Opts). @@ -257,7 +258,7 @@ start(ResId, Opts) -> restart(ResId) -> restart(ResId, #{}). --spec restart(resource_id(), create_opts()) -> ok | {error, Reason :: term()}. +-spec restart(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}. restart(ResId, Opts) -> emqx_resource_manager:restart(ResId, Opts). @@ -277,6 +278,25 @@ set_resource_status_connecting(ResId) -> get_instance(ResId) -> emqx_resource_manager:lookup(ResId). +-spec fetch_creation_opts(map()) -> creation_opts(). +fetch_creation_opts(Opts) -> + SupportedOpts = [ + health_check_interval, + health_check_timeout, + wait_for_resource_ready, + start_after_created, + auto_retry_interval, + enable_batch, + batch_size, + batch_time, + enable_queue, + queue_max_bytes, + query_mode, + resume_interval, + async_inflight_window + ], + maps:with(creation_opts(), SupportedOpts). + -spec list_instances() -> [resource_id()]. list_instances() -> [Id || #{id := Id} <- list_instances_verbose()]. @@ -341,7 +361,7 @@ check_and_create(ResId, Group, ResourceType, RawConfig) -> resource_group(), resource_type(), raw_resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data() | 'already_created'} | {error, term()}. check_and_create(ResId, Group, ResourceType, RawConfig, Opts) -> @@ -366,7 +386,7 @@ check_and_create_local(ResId, Group, ResourceType, RawConfig) -> resource_group(), resource_type(), raw_resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data()} | {error, term()}. check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) -> check_and_do( @@ -379,7 +399,7 @@ check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) -> resource_id(), resource_type(), raw_resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data()} | {error, term()}. check_and_recreate(ResId, ResourceType, RawConfig, Opts) -> @@ -393,7 +413,7 @@ check_and_recreate(ResId, ResourceType, RawConfig, Opts) -> resource_id(), resource_type(), raw_resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data()} | {error, term()}. check_and_recreate_local(ResId, ResourceType, RawConfig, Opts) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 3310555d1..608548898 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -85,7 +85,7 @@ manager_id_to_resource_id(MgrId) -> resource_group(), resource_type(), resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data()}. ensure_resource(ResId, Group, ResourceType, Config, Opts) -> case lookup(ResId) of @@ -97,7 +97,7 @@ ensure_resource(ResId, Group, ResourceType, Config, Opts) -> end. %% @doc Called from emqx_resource when recreating a resource which may or may not exist --spec recreate(resource_id(), resource_type(), resource_config(), create_opts()) -> +-spec recreate(resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, not_found} | {error, updating_to_incorrect_resource_type}. recreate(ResId, ResourceType, NewConfig, Opts) -> case lookup(ResId) of @@ -166,7 +166,7 @@ remove(ResId, ClearMetrics) when is_binary(ResId) -> safe_call(ResId, {remove, ClearMetrics}, ?T_OPERATION). %% @doc Stops and then starts an instance that was already running --spec restart(resource_id(), create_opts()) -> ok | {error, Reason :: term()}. +-spec restart(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}. restart(ResId, Opts) when is_binary(ResId) -> case safe_call(ResId, restart, ?T_OPERATION) of ok -> @@ -177,7 +177,7 @@ restart(ResId, Opts) when is_binary(ResId) -> end. %% @doc Start the resource --spec start(resource_id(), create_opts()) -> ok | {error, Reason :: term()}. +-spec start(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}. start(ResId, Opts) -> case safe_call(ResId, start, ?T_OPERATION) of ok -> diff --git a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl index cdd2592d9..11af1a62c 100644 --- a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl +++ b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl @@ -38,7 +38,7 @@ introduced_in() -> resource_group(), resource_type(), resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. create(ResId, Group, ResourceType, Config, Opts) -> @@ -58,7 +58,7 @@ create_dry_run(ResourceType, Config) -> resource_id(), resource_type(), resource_config(), - create_opts() + creation_opts() ) -> {ok, resource_data()} | {error, Reason :: term()}. recreate(ResId, ResourceType, Config, Opts) -> diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 933cd0189..464c055b7 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -30,22 +30,14 @@ namespace() -> "resource_schema". roots() -> []. -fields('batch&async&queue') -> +fields('creation_opts') -> [ {query_mode, fun query_mode/1}, {resume_interval, fun resume_interval/1}, {async_inflight_window, fun async_inflight_window/1}, - {batch, mk(ref(?MODULE, batch), #{desc => ?DESC("batch")})}, - {queue, mk(ref(?MODULE, queue), #{desc => ?DESC("queue")})} - ]; -fields(batch) -> - [ {enable_batch, fun enable_batch/1}, {batch_size, fun batch_size/1}, - {batch_time, fun batch_time/1} - ]; -fields(queue) -> - [ + {batch_time, fun batch_time/1}, {enable_queue, fun enable_queue/1}, {max_queue_bytes, fun queue_max_bytes/1} ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 4edeb786a..23c6788e8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -111,7 +111,7 @@ fields(Name) when Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2 -> fields(basic) ++ - emqx_resource_schema:fields('batch&async&queue') ++ + emqx_resource_schema:fields('creation_opts') ++ connector_field(Name). method_fileds(post, ConnectorType) -> From 3a76a50382a55ddc1b394bd9d3d671c56b0e2968 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 19:45:52 +0800 Subject: [PATCH 52/71] fix: syntax error and compile error --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 2 +- apps/emqx_resource/include/emqx_resource.hrl | 2 +- apps/emqx_resource/src/emqx_resource.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index ac1ec6ba3..66c4524b0 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -34,7 +34,7 @@ create_dry_run/2, remove/1, remove/2, - remove/3, + remove/4, update/2, update/3, stop/2, diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 5b6856dc0..ed8929831 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -58,7 +58,7 @@ batch_time => integer(), enable_queue => boolean(), queue_max_bytes => integer(), - query_mode => async | sync | dynamic + query_mode => async | sync | dynamic, resume_interval => integer(), async_inflight_window => integer() }. diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b134d2af1..c4fd24007 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -295,7 +295,7 @@ fetch_creation_opts(Opts) -> resume_interval, async_inflight_window ], - maps:with(creation_opts(), SupportedOpts). + maps:with(SupportedOpts, Opts). -spec list_instances() -> [resource_id()]. list_instances() -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 23c6788e8..ece2a6bc7 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -71,7 +71,9 @@ values(Protocol, post) -> <<"${topic},clientid=${clientid}", " ", "payload=${payload},", "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary, "bool=${payload.bool}">>, - batch => #{enable_batch => false, batch_size => 5, batch_time => <<"1m">>} + enable_batch => false, + batch_size => 5, + batch_time => <<"1m">> }; values(Protocol, put) -> values(Protocol, post). From db3e4f0b906aa811c6fec268381f15fc9a2ff2ec Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 20:58:56 +0800 Subject: [PATCH 53/71] fix: flatten Points list --- .../emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 41f7059ec..0a2bbb638 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -389,13 +389,13 @@ to_maps_config(K, V, Res) -> %% Tags & Fields Data Trans parse_batch_data(InstId, BatchData, SyntaxLines) -> {Points, Errors} = lists:foldl( - fun({send_message, Data}, {AccIn, ErrAccIn}) -> + fun({send_message, Data}, {ListOfPoints, ErrAccIn}) -> case data_to_points(Data, SyntaxLines) of {ok, Points} -> - {[Points | AccIn], ErrAccIn}; + {[Points | ListOfPoints], ErrAccIn}; {error, ErrorPoints} -> log_error_points(InstId, ErrorPoints), - {AccIn, ErrAccIn + 1} + {ListOfPoints, ErrAccIn + 1} end end, {[], 0}, @@ -403,7 +403,7 @@ parse_batch_data(InstId, BatchData, SyntaxLines) -> ), case Errors of 0 -> - {ok, Points}; + {ok, lists:flatten(Points)}; _ -> ?SLOG(error, #{ msg => io_lib:format("InfluxDB trans point failed, count: ~p", [Errors]), From 88388b0c54dabb38ad3e55ae9a540652705b7fd7 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 11 Aug 2022 18:13:27 +0800 Subject: [PATCH 54/71] feat: http connector support async sending --- .../{ => schema}/emqx_bridge_mqtt_schema.erl | 0 .../src/{ => schema}/emqx_bridge_schema.erl | 14 ++--- .../emqx_bridge_webhook_schema.erl | 0 .../src/emqx_connector_http.erl | 52 +++++++++++++++++-- mix.exs | 2 +- rebar.config | 2 +- 6 files changed, 58 insertions(+), 12 deletions(-) rename apps/emqx_bridge/src/{ => schema}/emqx_bridge_mqtt_schema.erl (100%) rename apps/emqx_bridge/src/{ => schema}/emqx_bridge_schema.erl (96%) rename apps/emqx_bridge/src/{ => schema}/emqx_bridge_webhook_schema.erl (100%) diff --git a/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl similarity index 100% rename from apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl rename to apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl similarity index 96% rename from apps/emqx_bridge/src/emqx_bridge_schema.erl rename to apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 46039390d..4343dc223 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -27,15 +27,15 @@ %%====================================================================================== %% For HTTP APIs get_response() -> - http_schema("get"). + api_schema("get"). put_request() -> - http_schema("put"). + api_schema("put"). post_request() -> - http_schema("post"). + api_schema("post"). -http_schema(Method) -> +api_schema(Method) -> Broker = lists:flatmap( fun(Type) -> @@ -46,17 +46,17 @@ http_schema(Method) -> end, ?CONN_TYPES ) ++ [ref(Module, Method) || Module <- [emqx_bridge_webhook_schema]], - EE = ee_schemas(Method), + EE = ee_api_schemas(Method), hoconsc:union(Broker ++ EE). -if(?EMQX_RELEASE_EDITION == ee). -ee_schemas(Method) -> +ee_api_schemas(Method) -> emqx_ee_bridge:api_schemas(Method). ee_fields_bridges() -> emqx_ee_bridge:fields(bridges). -else. -ee_schemas(_) -> +ee_api_schemas(_) -> []. ee_fields_bridges() -> diff --git a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl similarity index 100% rename from apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl rename to apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index c5a1b89db..11030fef6 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -30,6 +30,7 @@ on_start/2, on_stop/2, on_query/3, + on_query_async/4, on_get_status/2 ]). @@ -165,7 +166,7 @@ ref(Field) -> hoconsc:ref(?MODULE, Field). %% =================================================================== -callback_mode() -> always_sync. +callback_mode() -> async_if_possible. on_start( InstId, @@ -231,7 +232,8 @@ on_stop(InstId, #{pool_name := PoolName}) -> on_query(InstId, {send_message, Msg}, State) -> case maps:get(request, State, undefined) of undefined -> - ?SLOG(error, #{msg => "request_not_found", connector => InstId}); + ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}), + {error, arg_request_not_found}; Request -> #{ method := Method, @@ -302,6 +304,51 @@ on_query( end, Result. +on_query_async(InstId, {send_message, Msg}, ReplyFun, State) -> + case maps:get(request, State, undefined) of + undefined -> + ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}), + {error, arg_request_not_found}; + Request -> + #{ + method := Method, + path := Path, + body := Body, + headers := Headers, + request_timeout := Timeout + } = process_request(Request, Msg), + on_query_async( + InstId, + {undefined, Method, {Path, Headers, Body}, Timeout}, + ReplyFun, + State + ) + end; +on_query_async( + InstId, + {KeyOrNum, Method, Request, Timeout}, + ReplyFun, + #{pool_name := PoolName, base_path := BasePath} = State +) -> + ?TRACE( + "QUERY_ASYNC", + "http_connector_received", + #{request => Request, connector => InstId, state => State} + ), + NRequest = formalize_request(Method, BasePath, Request), + Worker = + case KeyOrNum of + undefined -> ehttpc_pool:pick_worker(PoolName); + _ -> ehttpc_pool:pick_worker(PoolName, KeyOrNum) + end, + ok = ehttpc:request_async( + Worker, + Method, + NRequest, + Timeout, + ReplyFun + ). + on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) -> case do_get_status(PoolName, Timeout) of true -> @@ -343,7 +390,6 @@ do_get_status(PoolName, Timeout) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- - preprocess_request(undefined) -> undefined; preprocess_request(Req) when map_size(Req) == 0 -> diff --git a/mix.exs b/mix.exs index 5209de843..2e4765ce1 100644 --- a/mix.exs +++ b/mix.exs @@ -47,7 +47,7 @@ defmodule EMQXUmbrella.MixProject do {:lc, github: "emqx/lc", tag: "0.3.1"}, {:redbug, "2.0.7"}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, - {:ehttpc, github: "emqx/ehttpc", tag: "0.3.0", override: true}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.4.0", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, diff --git a/rebar.config b/rebar.config index 3d3d5968f..bc5c8d396 100644 --- a/rebar.config +++ b/rebar.config @@ -49,7 +49,7 @@ , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.7"}}} - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.3.0"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.0"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} From ed796acb9558ca7ec74b203c7a42903f10d04933 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 22:11:28 +0800 Subject: [PATCH 55/71] fix: fetch resource options after reboot --- apps/emqx_bridge/src/emqx_bridge.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index c794a25a5..08939d915 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -49,14 +49,14 @@ -export([get_basic_usage_info/0]). load() -> - %% set wait_for_resource_ready => 0 to start resources async - Opts = #{auto_retry_interval => 60000, wait_for_resource_ready => 0}, Bridges = emqx:get_config([bridges], #{}), lists:foreach( fun({Type, NamedConf}) -> lists:foreach( fun({Name, Conf}) -> - safe_load_bridge(Type, Name, Conf, Opts) + %% fetch opts for `emqx_resource_worker` + ResOpts = emqx_resource:fetch_creation_opts(Conf), + safe_load_bridge(Type, Name, Conf, ResOpts) end, maps:to_list(NamedConf) ) From 458dab53c5434fe2c9e034e48b100e2c8793608a Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 22:54:32 +0800 Subject: [PATCH 56/71] fix: undefined_functions dialyzer warning --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 66c4524b0..d34c30ee9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -37,6 +37,7 @@ remove/4, update/2, update/3, + update/4, stop/2, restart/2, reset_metrics/1 @@ -111,6 +112,9 @@ update(BridgeId, {OldConf, Conf}) -> update(BridgeType, BridgeName, {OldConf, Conf}). update(Type, Name, {OldConf, Conf}) -> + update(Type, Name, {OldConf, Conf}, #{}). + +update(Type, Name, {OldConf, Conf}, Opts) -> %% TODO: sometimes its not necessary to restart the bridge connection. %% %% - if the connection related configs like `servers` is updated, we should restart/start @@ -127,7 +131,7 @@ update(Type, Name, {OldConf, Conf}) -> name => Name, config => Conf }), - case recreate(Type, Name, Conf) of + case recreate(Type, Name, Conf, Opts) of {ok, _} -> maybe_disable_bridge(Type, Name, Conf); {error, not_found} -> @@ -137,7 +141,7 @@ update(Type, Name, {OldConf, Conf}) -> name => Name, config => Conf }), - create(Type, Name, Conf); + create(Type, Name, Conf, Opts); {error, Reason} -> {error, {update_bridge_failed, Reason}} end; From 83746daad529e08131774f795e04b392d6ef5ce8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 11 Aug 2022 22:59:27 +0800 Subject: [PATCH 57/71] fix: update bridge config badmap error --- apps/emqx_bridge/src/emqx_bridge.erl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 08939d915..354c4faee 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -260,6 +260,13 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> fun ({_Type, _Name}, _Conf, {error, Reason}) -> {error, Reason}; + %% for emqx_bridge_resource:update/4 + ({Type, Name}, {OldConf, Conf}, _) -> + ResOpts = emqx_resource:fetch_creation_opts(Conf), + case Action(Type, Name, {OldConf, Conf}, ResOpts) of + {error, Reason} -> {error, Reason}; + Return -> Return + end; ({Type, Name}, Conf, _) -> ResOpts = emqx_resource:fetch_creation_opts(Conf), case Action(Type, Name, Conf, ResOpts) of From c3c4ed02b4d4840f9b9081b21c0757eb1562dedb Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Aug 2022 00:24:58 +0800 Subject: [PATCH 58/71] fix: bump emqx_dashboard to 5.0.4 --- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index c3b7b4f13..4e1a3518f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index abdb220bb..9a2234ff5 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -45,7 +45,7 @@ emqx_resource_schema { async_inflight_window { desc { - en: """Async queyr inflight window.""" + en: """Async query inflight window.""" zh: """异步请求飞行队列窗口大小""" } label { From bed10b097f7a43e1a120985907de7dbcbecef62b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Aug 2022 00:27:13 +0800 Subject: [PATCH 59/71] chore: release e5.0.0-beta.1 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 75d5852c9..bad7e8229 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.4"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). +-define(EMQX_RELEASE_EE, "5.0.0-beta.1"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 54298183fba32b5ef44f93da4a20ed772aa31559 Mon Sep 17 00:00:00 2001 From: Rory Z Date: Fri, 12 Aug 2022 09:50:50 +0800 Subject: [PATCH 60/71] ci: do not push enterprise image to aws ecr --- .github/workflows/build_and_push_docker_images.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index c54ae7bd9..1b724d9e3 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -121,6 +121,8 @@ jobs: - aws-arm64 - ubuntu-20.04 exclude: + - registry: 'public.ecr.aws' + profile: emqx-enterprise - arch: arm64 build_machine: ubuntu-20.04 - arch: amd64 From 0c1285cdab56098e606211d21f60675df42ad0b4 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 10:07:54 +0800 Subject: [PATCH 61/71] fix: bridge creation opts refs --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index ece2a6bc7..6a1b2677f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -90,7 +90,8 @@ fields(basic) -> {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, {write_syntax, fun write_syntax/1} - ]; + ] ++ + emqx_resource_schema:fields('creation_opts'); fields("post_udp") -> method_fileds(post, influxdb_udp); fields("post_api_v1") -> @@ -113,7 +114,6 @@ fields(Name) when Name == influxdb_udp orelse Name == influxdb_api_v1 orelse Name == influxdb_api_v2 -> fields(basic) ++ - emqx_resource_schema:fields('creation_opts') ++ connector_field(Name). method_fileds(post, ConnectorType) -> From 907d9a166acdd9a0b6f8b6db493fbfcab312f207 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 12 Aug 2022 10:20:15 +0800 Subject: [PATCH 62/71] test(gw): fix gateway flaky tests --- apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl | 2 +- apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 6bbd2135b..4dd5819b3 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -79,7 +79,7 @@ end_per_suite(Config) -> emqx_gateway_auth_ct:stop(), ok = emqx_authz_test_lib:restore_authorizers(), emqx_config:erase(gateway), - emqx_mgmt_api_test_util:end_suite([cowboy, emqx_authz, emqx_authn, emqx_gateway]), + emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_authz, emqx_conf]), Config. init_per_testcase(_Case, Config) -> diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index 54928a792..084fd9f57 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -103,7 +103,7 @@ init_per_suite(Config) -> end_per_suite(_) -> {ok, _} = emqx:remove_config([gateway, mqttsn]), - emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_auhtn, emqx_conf]). + emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]). restart_mqttsn_with_subs_resume_on() -> Conf = emqx:get_raw_config([gateway, mqttsn]), From 0cdf4b47f11264389a283b5ea164b1d0f35e352b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Aug 2022 11:33:19 +0800 Subject: [PATCH 63/71] feat: add more resource creation opts --- apps/emqx_authn/src/emqx_authn_utils.erl | 1 - apps/emqx_authz/src/emqx_authz_utils.erl | 1 - apps/emqx_bridge/src/emqx_bridge_resource.erl | 4 +- .../src/schema/emqx_bridge_webhook_schema.erl | 18 +++++- .../i18n/emqx_resource_schema_i18n.conf | 56 ++++++++++++++++++- apps/emqx_resource/include/emqx_resource.hrl | 33 ++++++----- apps/emqx_resource/src/emqx_resource.erl | 5 +- .../src/emqx_resource_manager.erl | 25 ++++----- .../src/schema/emqx_resource_schema.erl | 28 ++++++++++ 9 files changed, 136 insertions(+), 35 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 994f2f275..dca243cc1 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -47,7 +47,6 @@ ]). -define(DEFAULT_RESOURCE_OPTS, #{ - auto_retry_interval => 6000, start_after_created => false }). diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index dd6f66c7f..0048c0dc4 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -40,7 +40,6 @@ ]). -define(DEFAULT_RESOURCE_OPTS, #{ - auto_retry_interval => 6000, start_after_created => false }). diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index d34c30ee9..f7aeec30d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -89,7 +89,7 @@ create(BridgeId, Conf) -> create(BridgeType, BridgeName, Conf). create(Type, Name, Conf) -> - create(Type, Name, Conf, #{auto_retry_interval => 60000}). + create(Type, Name, Conf, #{}). create(Type, Name, Conf, Opts) -> ?SLOG(info, #{ @@ -169,7 +169,7 @@ recreate(Type, Name, Conf, Opts) -> resource_id(Type, Name), bridge_to_resource_type(Type), parse_confs(Type, Name, Conf), - Opts#{auto_retry_interval => 60000} + Opts ). create_dry_run(Type, Conf) -> diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl index d833e6ca8..d51c2edef 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_webhook_schema.erl @@ -50,7 +50,7 @@ basic_config() -> default => egress } )} - ] ++ + ] ++ webhook_creation_opts() ++ proplists:delete(base_url, emqx_connector_http:fields(config)). request_config() -> @@ -116,6 +116,22 @@ request_config() -> )} ]. +webhook_creation_opts() -> + Opts = emqx_resource_schema:fields(creation_opts), + lists:filter( + fun({K, _V}) -> + not lists:member(K, unsupported_opts()) + end, + Opts + ). + +unsupported_opts() -> + [ + enable_batch, + batch_size, + batch_time + ]. + %%====================================================================================== type_field() -> diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index 9a2234ff5..3ec170ebf 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -1,4 +1,58 @@ emqx_resource_schema { + + health_check_interval { + desc { + en: """Health check interval, in milliseconds.""" + zh: """健康检查间隔，单位毫秒。""" + } + label { + en: """Health Check Interval""" + zh: """健康检查间隔""" + } + } + + start_after_created { + desc { + en: """Whether start the resource right after created.""" + zh: """是否在创建资源后立即启动资源。""" + } + label { + en: """Start After Created""" + zh: """创建后立即启动""" + } + } + + start_timeout { + desc { + en: """ +If 'start_after_created' enabled, how long time do we wait for the +resource get started, in milliseconds. +""" + zh: """ +如果选择了创建后立即启动资源，此选项用来设置等待资源启动的超时时间，单位毫秒。 +""" + } + label { + en: """Start Timeout""" + zh: """启动超时时间""" + } + } + + auto_restart_interval { + desc { + en: """ +The auto restart interval after the resource is disconnected, in milliseconds. +""" + zh: """ +资源断开以后，自动重连的时间间隔，单位毫秒。 +""" + } + label { + en: """Auto Restart Interval""" + zh: """自动重连间隔""" + } + } + query_mode { desc { en: """Query mode. Optional 'sync/async', default 'sync'.""" @@ -6,7 +60,7 @@ emqx_resource_schema { } label { en: """query_mode""" - zh: """query_mode""" + zh: """请求模式""" } } diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index ed8929831..190c278ae 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -41,18 +41,25 @@ }. -type resource_group() :: binary(). -type creation_opts() :: #{ - health_check_interval => integer(), + %%======================================= Deprecated Opts: + %% use health_check_interval instead health_check_timeout => integer(), - %% We can choose to block the return of emqx_resource:start until - %% the resource connected, wait max to `wait_for_resource_ready` ms. + %% use start_timeout instead wait_for_resource_ready => integer(), + %% use auto_restart_interval instead + auto_retry_interval => integer(), + %%======================================= Deprecated Opts End + health_check_interval => integer(), + %% We can choose to block the return of emqx_resource:start until + %% the resource connected, wait max to `start_timeout` ms. + start_timeout => integer(), %% If `start_after_created` is set to true, the resource is started right %% after it is created. But note that a `started` resource is not guaranteed %% to be `connected`. start_after_created => boolean(), %% If the resource disconnected, we can set to retry starting the resource %% periodically. - auto_retry_interval => integer(), + auto_restart_interval => integer(), enable_batch => boolean(), batch_size => integer(), batch_time => integer(), @@ -68,17 +75,17 @@ | {error, term()} | {resource_down, term()}. -%% count --define(DEFAULT_BATCH_SIZE, 100). -%% milliseconds --define(DEFAULT_BATCH_TIME, 10). - -%% bytes -define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024). - +-define(DEFAULT_BATCH_SIZE, 100). +-define(DEFAULT_BATCH_TIME, 10). -define(DEFAULT_INFLIGHT, 100). - +-define(HEALTHCHECK_INTERVAL, 15000). +-define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>). -define(RESUME_INTERVAL, 15000). - +-define(START_AFTER_CREATED, true). +-define(START_TIMEOUT, 5000). +-define(START_TIMEOUT_RAW, <<"5s">>). +-define(AUTO_RESTART_INTERVAL, 60000). +-define(AUTO_RESTART_INTERVAL_RAW, <<"60s">>). -define(TEST_ID_PREFIX, "_test_:"). -define(RES_METRICS, resource_metrics). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index c4fd24007..60f0dd360 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -282,10 +282,9 @@ get_instance(ResId) -> fetch_creation_opts(Opts) -> SupportedOpts = [ health_check_interval, - health_check_timeout, - wait_for_resource_ready, + start_timeout, start_after_created, - auto_retry_interval, + auto_restart_interval, enable_batch, batch_size, batch_time, diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 608548898..66d9e32b0 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -57,7 +57,6 @@ -type data() :: #data{}. -define(SHORT_HEALTHCHECK_INTERVAL, 1000). --define(HEALTHCHECK_INTERVAL, 15000). -define(ETS_TABLE, ?MODULE). -define(WAIT_FOR_RESOURCE_DELAY, 100). -define(T_OPERATION, 5000). @@ -127,9 +126,9 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) -> [matched] ), ok = emqx_resource_worker_sup:start_workers(ResId, Opts), - case maps:get(start_after_created, Opts, true) of + case maps:get(start_after_created, Opts, ?START_AFTER_CREATED) of true -> - wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)); + wait_for_ready(ResId, maps:get(start_timeout, Opts, ?START_TIMEOUT)); false -> ok end, @@ -147,7 +146,7 @@ create_dry_run(ResourceType, Config) -> ok = emqx_resource_manager_sup:ensure_child( MgrId, ResId, <<"dry_run">>, ResourceType, Config, #{} ), - case wait_for_resource_ready(ResId, 15000) of + case wait_for_ready(ResId, 15000) of ok -> remove(ResId); timeout -> @@ -170,7 +169,7 @@ remove(ResId, ClearMetrics) when is_binary(ResId) -> restart(ResId, Opts) when is_binary(ResId) -> case safe_call(ResId, restart, ?T_OPERATION) of ok -> - wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)), + wait_for_ready(ResId, maps:get(start_timeout, Opts, 5000)), ok; {error, _Reason} = Error -> Error @@ -181,7 +180,7 @@ restart(ResId, Opts) when is_binary(ResId) -> start(ResId, Opts) -> case safe_call(ResId, start, ?T_OPERATION) of ok -> - wait_for_resource_ready(ResId, maps:get(wait_for_resource_ready, Opts, 5000)), + wait_for_ready(ResId, maps:get(start_timeout, Opts, 5000)), ok; {error, _Reason} = Error -> Error @@ -422,7 +421,7 @@ get_owner(ResId) -> end. handle_disconnected_state_enter(Data) -> - case maps:get(auto_retry_interval, Data#data.opts, undefined) of + case maps:get(auto_restart_interval, Data#data.opts, ?AUTO_RESTART_INTERVAL) of undefined -> {next_state, disconnected, Data}; RetryInterval -> @@ -573,19 +572,19 @@ data_record_to_external_map_with_metrics(Data) -> metrics => get_metrics(Data#data.id) }. --spec wait_for_resource_ready(resource_id(), integer()) -> ok | timeout. -wait_for_resource_ready(ResId, WaitTime) -> - do_wait_for_resource_ready(ResId, WaitTime div ?WAIT_FOR_RESOURCE_DELAY). +-spec wait_for_ready(resource_id(), integer()) -> ok | timeout. +wait_for_ready(ResId, WaitTime) -> + do_wait_for_ready(ResId, WaitTime div ?WAIT_FOR_RESOURCE_DELAY). -do_wait_for_resource_ready(_ResId, 0) -> +do_wait_for_ready(_ResId, 0) -> timeout; -do_wait_for_resource_ready(ResId, Retry) -> +do_wait_for_ready(ResId, Retry) -> case ets_lookup(ResId) of {ok, _Group, #{status := connected}} -> ok; _ -> timer:sleep(?WAIT_FOR_RESOURCE_DELAY), - do_wait_for_resource_ready(ResId, Retry - 1) + do_wait_for_ready(ResId, Retry - 1) end. safe_call(ResId, Message, Timeout) -> diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 464c055b7..ccc31a707 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -32,6 +32,10 @@ roots() -> []. fields('creation_opts') -> [ + {health_check_interval, fun health_check_interval/1}, + {start_after_created, fun start_after_created/1}, + {start_timeout, fun start_timeout/1}, + {auto_restart_interval, fun auto_restart_interval/1}, {query_mode, fun query_mode/1}, {resume_interval, fun resume_interval/1}, {async_inflight_window, fun async_inflight_window/1}, @@ -42,6 +46,30 @@ fields('creation_opts') -> {max_queue_bytes, fun queue_max_bytes/1} ]. +health_check_interval(type) -> emqx_schema:duration_ms(); +health_check_interval(desc) -> ?DESC("health_check_interval"); +health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW; +health_check_interval(required) -> false; +health_check_interval(_) -> undefined. + +start_after_created(type) -> boolean(); +start_after_created(required) -> false; +start_after_created(default) -> ?START_AFTER_CREATED; +start_after_created(desc) -> ?DESC("start_after_created"); +start_after_created(_) -> undefined. + +start_timeout(type) -> emqx_schema:duration_ms(); +start_timeout(desc) -> ?DESC("start_timeout"); +start_timeout(default) -> ?START_TIMEOUT_RAW; +start_timeout(required) -> false; +start_timeout(_) -> undefined. + +auto_restart_interval(type) -> hoconsc:union([infinity, emqx_schema:duration_ms()]); +auto_restart_interval(desc) -> ?DESC("auto_restart_interval"); +auto_restart_interval(default) -> ?AUTO_RESTART_INTERVAL_RAW; +auto_restart_interval(required) -> false; +auto_restart_interval(_) -> undefined. + query_mode(type) -> enum([sync, async]); query_mode(desc) -> ?DESC("query_mode"); query_mode(default) -> sync; From 3678673124ec72aabd3564386c0cab0374fb4a57 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 10:44:47 +0800 Subject: [PATCH 64/71] fix: schema default value using raw type before convert --- apps/emqx_resource/include/emqx_resource.hrl | 19 +++++++++++++++++++ .../src/schema/emqx_resource_schema.erl | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 190c278ae..8ec57a00e 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -76,16 +76,35 @@ | {resource_down, term()}. -define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024). +-define(DEFAULT_QUEUE_SIZE_RAW, <<"1GB">>). + +%% count -define(DEFAULT_BATCH_SIZE, 100). + +%% milliseconds -define(DEFAULT_BATCH_TIME, 10). +-define(DEFAULT_BATCH_TIME_RAW, <<"10ms">>). + +%% count -define(DEFAULT_INFLIGHT, 100). + +%% milliseconds -define(HEALTHCHECK_INTERVAL, 15000). -define(HEALTHCHECK_INTERVAL_RAW, <<"15s">>). + +%% milliseconds -define(RESUME_INTERVAL, 15000). +-define(RESUME_INTERVAL_RAW, <<"15s">>). + -define(START_AFTER_CREATED, true). + +%% milliseconds -define(START_TIMEOUT, 5000). -define(START_TIMEOUT_RAW, <<"5s">>). + +%% milliseconds -define(AUTO_RESTART_INTERVAL, 60000). -define(AUTO_RESTART_INTERVAL_RAW, <<"60s">>). + -define(TEST_ID_PREFIX, "_test_:"). -define(RES_METRICS, resource_metrics). diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index ccc31a707..6111543d2 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -90,7 +90,7 @@ enable_queue(_) -> undefined. resume_interval(type) -> emqx_schema:duration_ms(); resume_interval(desc) -> ?DESC("resume_interval"); -resume_interval(default) -> ?RESUME_INTERVAL; +resume_interval(default) -> ?RESUME_INTERVAL_RAW; resume_interval(required) -> false; resume_interval(_) -> undefined. @@ -108,12 +108,12 @@ batch_size(_) -> undefined. batch_time(type) -> emqx_schema:duration_ms(); batch_time(desc) -> ?DESC("batch_time"); -batch_time(default) -> ?DEFAULT_BATCH_TIME; +batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW; batch_time(required) -> false; batch_time(_) -> undefined. queue_max_bytes(type) -> emqx_schema:bytesize(); queue_max_bytes(desc) -> ?DESC("queue_max_bytes"); -queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE; +queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; queue_max_bytes(required) -> false; queue_max_bytes(_) -> undefined. From fa5e8f142291547e901e42b87b31c016fe4ba45c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 11:31:21 +0800 Subject: [PATCH 65/71] chore: refine i18n label --- .../i18n/emqx_resource_schema_i18n.conf | 16 ++++++++-------- .../i18n/emqx_ee_bridge_influxdb.conf | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index 3ec170ebf..aa6579bbd 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -59,7 +59,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """请求模式。可选 '同步/异步'，默认为'同步'模式。""" } label { - en: """query_mode""" + en: """Query mode""" zh: """请求模式""" } } @@ -70,7 +70,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """启用批量模式。""" } label { - en: """enable_batch""" + en: """Enable batch""" zh: """启用批量模式""" } } @@ -81,7 +81,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """启用队列模式。""" } label { - en: """enable_queue""" + en: """Enable queue""" zh: """启用队列模式""" } } @@ -92,7 +92,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """资源不可用时的重试时间""" } label { - en: """resume_interval""" + en: """Resume interval""" zh: """恢复时间""" } } @@ -103,7 +103,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """异步请求飞行队列窗口大小""" } label { - en: """async_inflight_window""" + en: """Async inflight window""" zh: """异步请求飞行队列窗口""" } } @@ -114,7 +114,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """批量请求大小""" } label { - en: """batch_size""" + en: """Batch size""" zh: """批量请求大小""" } } @@ -125,7 +125,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """最大批量请求等待时间。""" } label { - en: """batch_time""" + en: """Batch time""" zh: """批量等待间隔""" } } @@ -136,7 +136,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. zh: """消息队列的最大长度，以字节计。""" } label { - en: """queue_max_bytes""" + en: """Queue max bytes""" zh: """队列最大长度""" } } diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf index ffd0b66a0..412922408 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf @@ -40,7 +40,7 @@ TLDR: """ } label { - en: "write_syntax" + en: "Write Syntax" zh: "写语句" } } @@ -57,7 +57,7 @@ TLDR: config_direction { desc { en: """The direction of this bridge, MUST be 'egress'""" - zh: """桥接的方向， 必须是 egress""" + zh: """桥接的方向，必须是 egress""" } label { en: "Bridge Direction" @@ -90,7 +90,7 @@ TLDR: desc_name { desc { en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" + zh: """桥接名字，人类可读的描述信息。""" } label { en: "Bridge Name" From dc7953c3e10e1c0dc792389fd15c45300ad4b764 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 17:28:56 +0800 Subject: [PATCH 66/71] chore: refine async query variable name --- apps/emqx_connector/src/emqx_connector_http.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 11030fef6..b7fd21c5f 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -304,7 +304,7 @@ on_query( end, Result. -on_query_async(InstId, {send_message, Msg}, ReplyFun, State) -> +on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) -> case maps:get(request, State, undefined) of undefined -> ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}), @@ -320,14 +320,14 @@ on_query_async(InstId, {send_message, Msg}, ReplyFun, State) -> on_query_async( InstId, {undefined, Method, {Path, Headers, Body}, Timeout}, - ReplyFun, + ReplyFunAndArgs, State ) end; on_query_async( InstId, {KeyOrNum, Method, Request, Timeout}, - ReplyFun, + ReplyFunAndArgs, #{pool_name := PoolName, base_path := BasePath} = State ) -> ?TRACE( @@ -346,7 +346,7 @@ on_query_async( Method, NRequest, Timeout, - ReplyFun + ReplyFunAndArgs ). on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) -> From 441d8c9d5750674db210e32fd188fcd2b3c59970 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 17:44:38 +0800 Subject: [PATCH 67/71] chore: bump `influxdb-client-erl` vsn --- lib-ee/emqx_ee_connector/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 485b0120d..5963b7ab0 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.3"}}} + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.4"}}} ]}. {shell, [ From 594d071c05ed243e1c6fb016fdd2e73e06f5dbf2 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 17:45:20 +0800 Subject: [PATCH 68/71] feat(influxdb): add async callback --- apps/emqx_resource/src/emqx_resource.erl | 8 +++++++ .../src/emqx_ee_connector_influxdb.erl | 24 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 60f0dd360..b79650904 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -114,6 +114,7 @@ -optional_callbacks([ on_query/3, on_batch_query/3, + on_query_async/4, on_get_status/2 ]). @@ -130,6 +131,13 @@ %% when calling emqx_resource:on_batch_query/3 -callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result(). +-callback on_query_async( + resource_id(), + Request :: term(), + {ReplyFun :: function(), Args :: list()}, + resource_state() +) -> query_result(). + %% when calling emqx_resource:health_check/2 -callback on_get_status(resource_id(), resource_state()) -> resource_status() diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 0a2bbb638..d0c17b6d5 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -18,6 +18,7 @@ on_stop/2, on_query/3, on_batch_query/3, + on_query_async/4, on_get_status/2 ]). @@ -62,6 +63,20 @@ on_batch_query(InstId, BatchData, State = #{write_syntax := SyntaxLines, client {resource_down, disconnected} end. +on_query_async( + InstId, + {send_message, Data}, + {ReplayFun, Args}, + _State = #{write_syntax := SyntaxLines, client := Client} +) -> + case data_to_points(Data, SyntaxLines) of + {ok, Points} -> + do_async_query(InstId, Client, Points, {ReplayFun, Args}); + {error, ErrorPoints} = Err -> + log_error_points(InstId, ErrorPoints), + Err + end. + on_get_status(_InstId, #{client := Client}) -> case influxdb:is_alive(Client) of true -> @@ -331,7 +346,6 @@ ssl_config(SSL = #{enable := true}) -> %% ------------------------------------------------------------------------------------------------- %% Query - do_query(InstId, Client, Points) -> case influxdb:write(Client, Points) of ok -> @@ -349,6 +363,14 @@ do_query(InstId, Client, Points) -> Err end. +do_async_query(InstId, Client, Points, ReplayFunAndArgs) -> + ?SLOG(info, #{ + msg => "influxdb write point async", + connector => InstId, + points => Points + }), + ok = influxdb:write_async(Client, Points, ReplayFunAndArgs). + %% ------------------------------------------------------------------------------------------------- %% Tags & Fields Config Trans From b01ae8ece6a80870d538aac6362d5cff6f76faf0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 15 Aug 2022 11:57:02 +0800 Subject: [PATCH 69/71] chore: refine influxdb bridge/connector i18n --- .../i18n/emqx_resource_schema_i18n.conf | 23 +++++--------- .../i18n/emqx_ee_bridge_influxdb.conf | 30 ++++++++----------- .../i18n/emqx_ee_connector_influxdb.conf | 16 +++++----- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index aa6579bbd..c07573b1a 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -24,13 +24,8 @@ emqx_resource_schema { start_timeout { desc { - en: """ -If 'start_after_created' enabled, how long time do we wait for the -resource get started, in milliseconds. -""" - zh: """ -如果选择了创建后立即启动资源，此选项用来设置等待资源启动的超时时间，单位毫秒。 -""" + en: """If 'start_after_created' enabled, how long time do we wait for the resource get started, in milliseconds.""" + zh: """如果选择了创建后立即启动资源，此选项用来设置等待资源启动的超时时间，单位毫秒。""" } label { en: """Start Timeout""" @@ -40,12 +35,8 @@ resource get started, in milliseconds. auto_restart_interval { desc { - en: """ -The auto restart interval after the resource is disconnected, in milliseconds. -""" - zh: """ -资源断开以后，自动重连的时间间隔，单位毫秒。 -""" + en: """The auto restart interval after the resource is disconnected, in milliseconds.""" + zh: """资源断开以后，自动重连的时间间隔，单位毫秒。""" } label { en: """Auto Restart Interval""" @@ -89,7 +80,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. resume_interval { desc { en: """Resume time interval when resource down.""" - zh: """资源不可用时的重试时间""" + zh: """资源不可用时的重试时间。""" } label { en: """Resume interval""" @@ -100,7 +91,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. async_inflight_window { desc { en: """Async query inflight window.""" - zh: """异步请求飞行队列窗口大小""" + zh: """异步请求飞行队列窗口大小。""" } label { en: """Async inflight window""" @@ -111,7 +102,7 @@ The auto restart interval after the resource is disconnected, in milliseconds. batch_size { desc { en: """Maximum batch count.""" - zh: """批量请求大小""" + zh: """批量请求大小。""" } label { en: """Batch size""" diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf index 412922408..9e805132e 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf @@ -1,15 +1,13 @@ emqx_ee_bridge_influxdb { local_topic { desc { - en: """ -The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic + en: """The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded. """ - zh: """ -发送到 'local_topic' 的消息都会转发到 InfluxDB。
+ zh: """发送到 'local_topic' 的消息都会转发到 InfluxDB。
注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。 """ } @@ -20,20 +18,18 @@ will be forwarded. } write_syntax { desc { - en: """ -Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. + en: """Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
-TLDR: +TLDR:
``` [,=[,=]] =[,=] [] ``` """ - zh: """ -使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符
+ zh: """使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符
参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及 [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
-TLDR: +TLDR:
``` [,=[,=]] =[,=] [] ``` @@ -46,8 +42,8 @@ TLDR: } config_enable { desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" + en: """Enable or disable this bridge.""" + zh: """启用/禁用桥接。""" } label { en: "Enable Or Disable Bridge" @@ -56,8 +52,8 @@ TLDR: } config_direction { desc { - en: """The direction of this bridge, MUST be 'egress'""" - zh: """桥接的方向，必须是 egress""" + en: """The direction of this bridge, MUST be 'egress'.""" + zh: """桥接的方向，必须是 egress。""" } label { en: "Bridge Direction" @@ -68,7 +64,7 @@ TLDR: desc_config { desc { en: """Configuration for an InfluxDB bridge.""" - zh: """InfluxDB 桥接配置""" + zh: """InfluxDB 桥接配置。""" } label: { en: "InfluxDB Bridge Configuration" @@ -78,8 +74,8 @@ TLDR: desc_type { desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" + en: """The Bridge Type.""" + zh: """Bridge 类型。""" } label { en: "Bridge Type" diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf index 7e223b9b7..4d2dc168c 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf @@ -42,8 +42,8 @@ emqx_ee_connector_influxdb { } protocol { desc { - en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2""" - zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2""" + en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2.""" + zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2。""" } label: { en: """Protocol""" @@ -53,7 +53,7 @@ emqx_ee_connector_influxdb { influxdb_udp { desc { en: """InfluxDB's UDP protocol.""" - zh: """InfluxDB UDP 协议""" + zh: """InfluxDB UDP 协议。""" } label: { en: """UDP Protocol""" @@ -63,7 +63,7 @@ emqx_ee_connector_influxdb { influxdb_api_v1 { desc { en: """InfluxDB's protocol. Support InfluxDB v1.8 and before.""" - zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本""" + zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本。""" } label: { en: """HTTP API Protocol""" @@ -73,7 +73,7 @@ emqx_ee_connector_influxdb { influxdb_api_v2 { desc { en: """InfluxDB's protocol. Support InfluxDB v2.0 and after.""" - zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本""" + zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本。""" } label: { en: """HTTP API V2 Protocol""" @@ -113,7 +113,7 @@ emqx_ee_connector_influxdb { bucket { desc { en: "InfluxDB bucket name." - zh: "InfluxDB bucket 名称" + zh: "InfluxDB bucket 名称。" } label: { en: "Bucket" @@ -152,8 +152,8 @@ emqx_ee_connector_influxdb { } pool_size { desc { - en: """InfluxDB Pool Size""" - zh: """InfluxDB 连接池大小""" + en: """InfluxDB Pool Size. Default value is CPU threads.""" + zh: """InfluxDB 连接池大小，默认为 CPU 线程数。""" } label { en: """InfluxDB Pool Size""" From 68946f1f6c5638d5c41272f69484858706fdb293 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 15 Aug 2022 11:57:37 +0800 Subject: [PATCH 70/71] feat: influxdb support `async`/`batch_async` query --- apps/emqx_resource/src/emqx_resource.erl | 10 +++++ .../src/emqx_ee_connector_influxdb.erl | 40 +++++++++++++------ 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b79650904..99e1f6057 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -115,6 +115,7 @@ on_query/3, on_batch_query/3, on_query_async/4, + on_batch_query_async/4, on_get_status/2 ]). @@ -131,6 +132,7 @@ %% when calling emqx_resource:on_batch_query/3 -callback on_batch_query(resource_id(), Request :: term(), resource_state()) -> query_result(). +%% when calling emqx_resource:on_query_async/4 -callback on_query_async( resource_id(), Request :: term(), @@ -138,6 +140,14 @@ resource_state() ) -> query_result(). +%% when calling emqx_resource:on_batch_query_async/4 +-callback on_batch_query_async( + resource_id(), + Request :: term(), + {ReplyFun :: function(), Args :: list()}, + resource_state() +) -> query_result(). + %% when calling emqx_resource:health_check/2 -callback on_get_status(resource_id(), resource_state()) -> resource_status() diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index d0c17b6d5..2c2de9a99 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -19,6 +19,7 @@ on_query/3, on_batch_query/3, on_query_async/4, + on_batch_query_async/4, on_get_status/2 ]). @@ -31,7 +32,7 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback -callback_mode() -> always_sync. +callback_mode() -> async_if_possible. on_start(InstId, Config) -> start_client(InstId, Config). @@ -50,17 +51,12 @@ on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, c %% Once a Batched Data trans to points failed. %% This batch query failed -on_batch_query(InstId, BatchData, State = #{write_syntax := SyntaxLines, client := Client}) -> - case on_get_status(InstId, State) of - connected -> - case parse_batch_data(InstId, BatchData, SyntaxLines) of - {ok, Points} -> - do_query(InstId, Client, Points); - {error, Reason} -> - {error, Reason} - end; - disconnected -> - {resource_down, disconnected} +on_batch_query(InstId, BatchData, _State = #{write_syntax := SyntaxLines, client := Client}) -> + case parse_batch_data(InstId, BatchData, SyntaxLines) of + {ok, Points} -> + do_query(InstId, Client, Points); + {error, Reason} -> + {error, Reason} end. on_query_async( @@ -77,6 +73,24 @@ on_query_async( Err end. +on_batch_query_async( + InstId, + BatchData, + {ReplayFun, Args}, + State = #{write_syntax := SyntaxLines, client := Client} +) -> + case on_get_status(InstId, State) of + connected -> + case parse_batch_data(InstId, BatchData, SyntaxLines) of + {ok, Points} -> + do_async_query(InstId, Client, Points, {ReplayFun, Args}); + {error, Reason} -> + {error, Reason} + end; + disconnected -> + {resource_down, disconnected} + end. + on_get_status(_InstId, #{client := Client}) -> case influxdb:is_alive(Client) of true -> @@ -122,7 +136,7 @@ fields(basic) -> mk(enum([ns, us, ms, s, m, h]), #{ required => false, default => ms, desc => ?DESC("precision") })}, - {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})} + {pool_size, mk(pos_integer(), #{desc => ?DESC("pool_size")})} ]; fields(influxdb_udp) -> fields(basic); From d0e923590e09df6ac0b913a78340e8e7eeffffcf Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 15 Aug 2022 14:01:02 +0800 Subject: [PATCH 71/71] fix: write influxdb line with undefined value --- .../emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 2c2de9a99..09a09aa44 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -502,7 +502,8 @@ maps_config_to_data(K, V, {Data, Res}) -> case {NK, NV} of {[undefined], _} -> {Data, Res}; - {_, [undefined]} -> + %% undefined value in normal format [undefined] or int/uint format [undefined, <<"i">>] + {_, [undefined | _]} -> {Data, Res}; _ -> {Data, Res#{NK => value_type(NV)}} @@ -512,7 +513,9 @@ value_type([Int, <<"i">>]) when is_integer(Int) -> {int, Int}; -value_type([UInt, <<"u">>]) -> +value_type([UInt, <<"u">>]) when + is_integer(UInt) +-> {uint, UInt}; value_type([<<"t">>]) -> 't';