From 54dedc8343c4e2fae5a83498fe1f598b2642f327 Mon Sep 17 00:00:00 2001 From: turtleDeng Date: Thu, 17 Jun 2021 19:10:38 +0800 Subject: [PATCH] Revert "chore: mv emqx_connector to emqx_data_bridge" This reverts commit d640e2ccfa68a1c0529ef302b3eadc627d07f5e5. --- apps/emqx_connector/.gitignore | 19 ++++++++++ apps/emqx_connector/README.md | 27 ++++++++++++++ apps/emqx_connector/etc/emqx_connector.conf | 4 +++ .../include/emqx_connector.hrl} | 0 .../emqx_connector/priv/emqx_connector.schema | 2 ++ apps/emqx_connector/rebar.config | 13 +++++++ .../emqx_connector/src/emqx_connector.app.src | 17 +++++++++ apps/emqx_connector/src/emqx_connector.erl | 16 +++++++++ .../emqx_connector/src/emqx_connector_app.erl | 31 ++++++++++++++++ .../src}/emqx_connector_ldap.erl | 2 +- .../src}/emqx_connector_mongo.erl | 2 +- .../src}/emqx_connector_mysql.erl | 0 .../src}/emqx_connector_pgsql.erl | 0 .../src}/emqx_connector_redis.erl | 2 +- .../src}/emqx_connector_schema_lib.erl | 2 +- .../emqx_connector/src/emqx_connector_sup.erl | 36 +++++++++++++++++++ .../priv/emqx_data_bridge.schema | 16 +++++++++ apps/emqx_data_bridge/rebar.config | 1 - .../src/emqx_data_bridge.app.src | 3 +- .../emqx_data_bridge/src/emqx_data_bridge.erl | 5 ++- apps/emqx_resource/priv/emqx_resource.schema | 2 ++ data/loaded_plugins.tmpl | 3 ++ rebar.config.erl | 9 +++-- 23 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 apps/emqx_connector/.gitignore create mode 100644 apps/emqx_connector/README.md create mode 100644 apps/emqx_connector/etc/emqx_connector.conf rename apps/{emqx_data_bridge/include/emqx_data_bridge.hrl => emqx_connector/include/emqx_connector.hrl} (100%) create mode 100644 apps/emqx_connector/priv/emqx_connector.schema create mode 100644 apps/emqx_connector/rebar.config create mode 100644 apps/emqx_connector/src/emqx_connector.app.src create mode 100644 apps/emqx_connector/src/emqx_connector.erl create mode 100644 apps/emqx_connector/src/emqx_connector_app.erl rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_ldap.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_mongo.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_mysql.erl (100%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_pgsql.erl (100%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_redis.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_schema_lib.erl (99%) create mode 100644 apps/emqx_connector/src/emqx_connector_sup.erl create mode 100644 apps/emqx_data_bridge/priv/emqx_data_bridge.schema create mode 100644 apps/emqx_resource/priv/emqx_resource.schema diff --git a/apps/emqx_connector/.gitignore b/apps/emqx_connector/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_connector/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md new file mode 100644 index 000000000..879669f93 --- /dev/null +++ b/apps/emqx_connector/README.md @@ -0,0 +1,27 @@ +# emqx_connector + +This application is a collection of `connectors`. + +A `connector` is a callback module of `emqx_resource` that maintains the data related to +external resources. Put all resource related callback modules in a single application is good as +we can put some util functions/modules here for reusing purpose. + +For example, a mysql connector is an emqx resource that maintains all the mysql connection +related parameters (configs) and the TCP connections to the mysql server. + +An mysql connector can be used as following: + +``` +(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). +[#{config => + #{auto_reconnect => true,cacertfile => [],certfile => [], + database => "mqtt",keyfile => [],password => "public", + pool_size => 1, + server => {{127,0,0,1},3306}, + ssl => false,user => "root",verify => false}, + id => <<"mysql-abc">>,mod => emqx_connector_mysql, + state => #{poolname => 'mysql-abc'}, + status => started}] +(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). +{ok,[<<"count(1)">>],[[1]]} +``` diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf new file mode 100644 index 000000000..db4402d47 --- /dev/null +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -0,0 +1,4 @@ +##-------------------------------------------------------------------- +## EMQ X CONNECTOR Plugin +##-------------------------------------------------------------------- + diff --git a/apps/emqx_data_bridge/include/emqx_data_bridge.hrl b/apps/emqx_connector/include/emqx_connector.hrl similarity index 100% rename from apps/emqx_data_bridge/include/emqx_data_bridge.hrl rename to apps/emqx_connector/include/emqx_connector.hrl diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema new file mode 100644 index 000000000..b8476c4d9 --- /dev/null +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx_connector config mapping diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config new file mode 100644 index 000000000..53b5c63e8 --- /dev/null +++ b/apps/emqx_connector/rebar.config @@ -0,0 +1,13 @@ +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_connector]} +]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src new file mode 100644 index 000000000..a821b8f13 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -0,0 +1,17 @@ +{application, emqx_connector, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_connector_app, []}}, + {applications, + [kernel, + stdlib, + emqx_resource, + ecpool + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl new file mode 100644 index 000000000..dd0359348 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector). diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl new file mode 100644 index 000000000..4bbad75cf --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_connector_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl rename to apps/emqx_connector/src/emqx_connector_ldap.erl index 322bce1ef..3b0af5aa7 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_ldap). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl rename to apps/emqx_connector/src/emqx_connector_mongo.erl index 9e22e5ac5..26a36dd0a 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mongo). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl similarity index 100% rename from apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl rename to apps/emqx_connector/src/emqx_connector_mysql.erl diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl similarity index 100% rename from apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl rename to apps/emqx_connector/src/emqx_connector_pgsql.erl diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl rename to apps/emqx_connector/src/emqx_connector_redis.erl index 1037afede..3eebbcf87 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_redis). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl rename to apps/emqx_connector/src/emqx_connector_schema_lib.erl index aaacd0e80..03572d91d 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_schema_lib). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -export([ relational_db_fields/0 diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl new file mode 100644 index 000000000..603b9a8ad --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema new file mode 100644 index 000000000..c9cb3f2c0 --- /dev/null +++ b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema @@ -0,0 +1,16 @@ +%%-*- mode: erlang -*- +%% emqx_data_bridge config mapping + +{mapping, "emqx_data_bridge.bridges", "emqx_data_bridge.bridges", [ + {default, []}, + {datatype, string} +]}. + +% fields("emqx_data_bridge") -> +% [ +% {bridges, +% [fun(mapping) -> "emqx_data_bridge.bridges"; +% (type) -> list(); +% (_) -> undefined +% end]} +% ] \ No newline at end of file diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_data_bridge/rebar.config index 2778ec974..cf4cfcf1b 100644 --- a/apps/emqx_data_bridge/rebar.config +++ b/apps/emqx_data_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. {deps, []}. -% {extra_src_dirs, [{"src", [{recursive, true}]}]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src index edfcd4a9d..360511d9b 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src @@ -5,8 +5,7 @@ {mod, {emqx_data_bridge_app, []}}, {applications, [kernel, - stdlib, - ecpool + stdlib ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index 54a228763..5877cd3dd 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -25,9 +25,8 @@ ]). load_bridges() -> - ConfFile = filename:join([emqx:get_env(plugins_etc_dir), ?MODULE]) ++ ".conf", - {ok, #{<<"emqx_data_bridge">> := RawConfig}} = hocon:load(ConfFile), - Bridges = maps:get(<<"bridges">>, RawConfig, []), + Bridges = proplists:get_value(bridges, + application:get_all_env(emqx_data_bridge), []), emqx_data_bridge_monitor:ensure_all_started(Bridges). resource_type(<<"mysql">>) -> emqx_connector_mysql; diff --git a/apps/emqx_resource/priv/emqx_resource.schema b/apps/emqx_resource/priv/emqx_resource.schema new file mode 100644 index 000000000..8246dc6a7 --- /dev/null +++ b/apps/emqx_resource/priv/emqx_resource.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx-resource config mapping diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index d0dac7fe1..5ac46e0e3 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -5,4 +5,7 @@ {emqx_retainer, {{enable_plugin_emqx_retainer}}}. {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. +{emqx_resource, {{enable_plugin_emqx_resource}}}. +{emqx_connector, {{enable_plugin_emqx_connector}}}. +{emqx_data_bridge, {{enable_plugin_emqx_data_bridge}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index 45baf0ea1..0248e6dab 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -186,6 +186,9 @@ overlay_vars_rel(RelType) -> end, [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} + , {enable_plugin_emqx_resource, true} + , {enable_plugin_emqx_connector, true} + , {enable_plugin_emqx_data_bridge, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -244,8 +247,6 @@ relx_apps(ReleaseType) -> , {emqx_plugin_libs, load} , observer_cli , emqx_http_lib - , emqx_resource - , emqx_data_bridge ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] @@ -283,8 +284,10 @@ relx_plugin_apps(ReleaseType) -> , emqx_auth_mnesia , emqx_web_hook , emqx_recon - , emqx_rule_engine + , emqx_resource + , emqx_connector , emqx_data_bridge + , emqx_rule_engine , emqx_sasl ] ++ [emqx_telemetry || not is_enterprise()]