From 6b33172095d181c512c17fb855ccf07874aa9c8a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 31 May 2021 10:21:38 +0800 Subject: [PATCH 01/19] feat(emqx_connector): add first emqx_connector for mysql --- apps/emqx_connector/.gitignore | 19 ++ apps/emqx_connector/LICENSE | 191 ++++++++++++++++++ apps/emqx_connector/README.md | 9 + apps/emqx_connector/rebar.config | 13 ++ .../emqx_connector/src/emqx_connector.app.src | 16 ++ .../emqx_connector/src/emqx_connector_app.erl | 20 ++ .../src/emqx_connector_mysql.erl | 68 +++++++ .../src/emqx_connector_schema_lib.erl | 102 ++++++++++ .../emqx_connector/src/emqx_connector_sup.erl | 35 ++++ .../src/emqx_plugin_libs_pool.erl | 57 ++++++ data/loaded_plugins.tmpl | 1 + rebar.config.erl | 2 + 12 files changed, 533 insertions(+) create mode 100644 apps/emqx_connector/.gitignore create mode 100644 apps/emqx_connector/LICENSE create mode 100644 apps/emqx_connector/README.md create mode 100644 apps/emqx_connector/rebar.config create mode 100644 apps/emqx_connector/src/emqx_connector.app.src create mode 100644 apps/emqx_connector/src/emqx_connector_app.erl create mode 100644 apps/emqx_connector/src/emqx_connector_mysql.erl create mode 100644 apps/emqx_connector/src/emqx_connector_schema_lib.erl create mode 100644 apps/emqx_connector/src/emqx_connector_sup.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl diff --git a/apps/emqx_connector/.gitignore b/apps/emqx_connector/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_connector/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_connector/LICENSE b/apps/emqx_connector/LICENSE new file mode 100644 index 000000000..2f97fdd03 --- /dev/null +++ b/apps/emqx_connector/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2021, Shawn <506895667@qq.com>. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md new file mode 100644 index 000000000..cf64b5b68 --- /dev/null +++ b/apps/emqx_connector/README.md @@ -0,0 +1,9 @@ +emqx_connector +===== + +An OTP application + +Build +----- + + $ rebar3 compile diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config new file mode 100644 index 000000000..53b5c63e8 --- /dev/null +++ b/apps/emqx_connector/rebar.config @@ -0,0 +1,13 @@ +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_connector]} +]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src new file mode 100644 index 000000000..16a1d0869 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -0,0 +1,16 @@ +{application, emqx_connector, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_connector_app, []}}, + {applications, + [kernel, + stdlib, + emqx_resource + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl new file mode 100644 index 000000000..402e94187 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -0,0 +1,20 @@ +%%%------------------------------------------------------------------- +%% @doc emqx_connector public API +%% @end +%%%------------------------------------------------------------------- + +-module(emqx_connector_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_connector_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl new file mode 100644 index 000000000..d9c414389 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -0,0 +1,68 @@ +-module(emqx_connector_mysql). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +-emqx_resource_api_path("connectors/mysql"). + +-export([fields/1]). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + ]). + +-export([do_health_check/1]). + +fields("config") -> + emqx_connector_schema_lib:relational_db_fields() ++ + emqx_connector_schema_lib:ssl_fields(). + +%% =================================================================== + +on_start(InstId, #{server := {Host, Port}, + database := DB, + user := User, + password := Password, + auto_reconnect := AutoReconn, + pool_size := PoolSize} = Config) -> + logger:info("starting mysql connector: ~p, config: ~p", [InstId, Config]), + SslOpts = case maps:get(ssl, Config) of + true -> + [{ssl, [{server_name_indication, disable} | + emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)]}]; + false -> + [] + end, + Options = [{host, Host}, + {port, Port}, + {user, User}, + {password, Password}, + {database, DB}, + {auto_reconnect, reconn_interval(AutoReconn)}, + {pool_size, PoolSize}], + PoolName = emqx_plugin_libs_pool:pool_name(InstId), + _ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts), + {ok, #{poolname => PoolName}}. + +on_stop(InstId, #{poolname := PoolName}) -> + logger:info("stopping mysql connector: ~p", [InstId]), + emqx_plugin_libs_pool:stop_pool(PoolName). + +on_query(InstId, Request, AfterQuery, State) -> + io:format("== the demo log tracer ~p received request: ~p~nstate: ~p~n", + [InstId, Request, State]), + emqx_resource:query_success(AfterQuery), + "this is a demo log messages...". + +on_health_check(_InstId, #{poolname := PoolName} = State) -> + emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State). + +do_health_check(Conn) -> + ok == element(1, mysql:query(Conn, <<"SELECT count(1) AS T">>)). + +%% =================================================================== +reconn_interval(true) -> 15; +reconn_interval(false) -> false. diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl new file mode 100644 index 000000000..8e69245b9 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -0,0 +1,102 @@ +-module(emqx_connector_schema_lib). +-include_lib("typerefl/include/types.hrl"). + +-export([ relational_db_fields/0 + , ssl_fields/0 + ]). + +-export([ to_ip_port/1 + ]). + +-typerefl_from_string({ip_port/0, emqx_connector_schema_lib, to_ip_port}). + +-reflect_type([ip_port/0]). + +-type ip_port() :: tuple(). + +-define(VALID, emqx_resource_validator). +-define(REQUIRED(MSG), ?VALID:required(MSG)). +-define(MAX(MAXV), ?VALID:max(number, MAXV)). +-define(MIN(MINV), ?VALID:min(number, MINV)). + +relational_db_fields() -> + [ {server, fun server/1} + , {database, fun database/1} + , {pool_size, fun pool_size/1} + , {user, fun user/1} + , {password, fun password/1} + , {auto_reconnect, fun auto_reconnect/1} + ]. + +ssl_fields() -> + [ {ssl, fun ssl/1} + , {cacertfile, fun cacertfile/1} + , {keyfile, fun keyfile/1} + , {certfile, fun certfile/1} + , {verify, fun verify/1} + ]. + +server(mapping) -> "config.server"; +server(type) -> ip_port(); +server(validator) -> [?REQUIRED("the field 'server' is required")]; +server(_) -> undefined. + +database(mapping) -> "config.database"; +database(type) -> string(); +database(validator) -> [?REQUIRED("the field 'server' is required")]; +database(_) -> undefined. + +pool_size(mapping) -> "config.pool_size"; +pool_size(type) -> integer(); +pool_size(default) -> 8; +pool_size(validator) -> [?MIN(1), ?MAX(64)]; +pool_size(_) -> undefined. + +user(mapping) -> "config.user"; +user(type) -> string(); +user(default) -> "root"; +user(_) -> undefined. + +password(mapping) -> "config.password"; +password(type) -> string(); +password(default) -> ""; +password(_) -> undefined. + +auto_reconnect(mapping) -> "config.auto_reconnect"; +auto_reconnect(type) -> boolean(); +auto_reconnect(default) -> true; +auto_reconnect(_) -> undefined. +ssl(mapping) -> "config.ssl"; +ssl(type) -> boolean(); +ssl(default) -> false; +ssl(_) -> undefined. + +cacertfile(mapping) -> "config.cacertfile"; +cacertfile(type) -> string(); +cacertfile(default) -> ""; +cacertfile(_) -> undefined. + +keyfile(mapping) -> "config.keyfile"; +keyfile(type) -> string(); +keyfile(default) -> ""; +keyfile(_) -> undefined. + +certfile(mapping) -> "config.certfile"; +certfile(type) -> string(); +certfile(default) -> ""; +certfile(_) -> undefined. + +verify(mapping) -> "config.verify"; +verify(type) -> boolean(); +verify(default) -> false; +verify(_) -> undefined. + +to_ip_port(Str) -> + case string:tokens(Str, ":") of + [Ip, Port] -> + case inet:parse_address(Ip) of + {ok, R} -> {ok, {R, list_to_integer(Port)}}; + _ -> {error, Str} + end; + _ -> {error, Str} + end. \ No newline at end of file diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl new file mode 100644 index 000000000..454dd7084 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -0,0 +1,35 @@ +%%%------------------------------------------------------------------- +%% @doc emqx_connector top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(emqx_connector_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl new file mode 100644 index 000000000..3db29b427 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_plugin_libs_pool). + +-export([ start_pool/3 + , stop_pool/1 + , pool_name/1 + , health_check/3 + ]). + +pool_name(ID) when is_binary(ID) -> + list_to_atom(binary_to_list(ID)). + +start_pool(Name, Mod, Options) -> + case ecpool:start_sup_pool(Name, Mod, Options) of + {ok, _} -> logger:log(info, "Initiated ~0p Successfully", [Name]); + {error, {already_started, _Pid}} -> + stop_pool(Name), + start_pool(Name, Mod, Options); + {error, Reason} -> + logger:log(error, "Initiate ~0p failed ~0p", [Name, Reason]), + error({start_pool_failed, Name}) + end. + +stop_pool(Name) -> + case ecpool:stop_sup_pool(Name) of + ok -> logger:log(info, "Destroyed ~0p Successfully", [Name]); + {error, Reason} -> + logger:log(error, "Destroy ~0p failed, ~0p", [Name, Reason]), + error({stop_pool_failed, Name}) + end. + +health_check(PoolName, CheckFunc, State) when is_function(CheckFunc) -> + Status = [begin + case ecpool_worker:client(Worker) of + {ok, Conn} -> CheckFunc(Conn); + _ -> false + end + end || {_WorkerName, Worker} <- ecpool:workers(PoolName)], + case length(Status) > 0 andalso lists:all(fun(St) -> St =:= true end, Status) of + true -> {ok, State}; + false -> {error, test_query_failed} + end. diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index 236eeaa95..eb4e58d1f 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -6,4 +6,5 @@ {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. {emqx_resource, {{enable_plugin_emqx_resource}}}. +{emqx_connector, {{enable_plugin_emqx_connector}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index 6e81908ba..fa5586ad5 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -181,6 +181,7 @@ overlay_vars_rel(RelType) -> [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} , {enable_plugin_emqx_resource, true} + , {enable_plugin_emqx_connector, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -275,6 +276,7 @@ relx_plugin_apps(ReleaseType) -> , emqx_web_hook , emqx_recon , emqx_resource + , emqx_connector , emqx_rule_engine , emqx_sasl ] From f4bb589079e2b93418d803365c17342daa956de4 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 31 May 2021 14:00:58 +0800 Subject: [PATCH 02/19] feat(emqx_mysql_connector): implement the on_query callback --- apps/emqx_connector/etc/emqx_connector.conf | 3 +++ .../emqx_connector/priv/emqx_connector.schema | 2 ++ .../src/emqx_connector_mysql.erl | 20 ++++++++++++++----- 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_connector/etc/emqx_connector.conf create mode 100644 apps/emqx_connector/priv/emqx_connector.schema diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf new file mode 100644 index 000000000..54b57ebe6 --- /dev/null +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -0,0 +1,3 @@ +##-------------------------------------------------------------------- +## EMQ X CONNECTOR Plugin +##-------------------------------------------------------------------- diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema new file mode 100644 index 000000000..b8476c4d9 --- /dev/null +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx_connector config mapping diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index d9c414389..ad9b9a4d1 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -14,6 +14,8 @@ , on_health_check/2 ]). +-export([connect/1]). + -export([do_health_check/1]). fields("config") -> @@ -51,11 +53,16 @@ on_stop(InstId, #{poolname := PoolName}) -> logger:info("stopping mysql connector: ~p", [InstId]), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, Request, AfterQuery, State) -> - io:format("== the demo log tracer ~p received request: ~p~nstate: ~p~n", - [InstId, Request, State]), - emqx_resource:query_success(AfterQuery), - "this is a demo log messages...". +on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> + logger:debug("mysql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), + case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL]}, no_handover) of + {error, Reason} -> + logger:debug("mysql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), + emqx_resource:query_failure(AfterQuery); + _ -> + emqx_resource:query_success(AfterQuery) + end, + Result. on_health_check(_InstId, #{poolname := PoolName} = State) -> emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State). @@ -66,3 +73,6 @@ do_health_check(Conn) -> %% =================================================================== reconn_interval(true) -> 15; reconn_interval(false) -> false. + +connect(Options) -> + mysql:start_link(Options). From e7ffa07a1a70e13eeb292c7a052764a18871d95b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 31 May 2021 22:49:42 +0800 Subject: [PATCH 03/19] feat(emqx_connector): load connectors from emqx_connector.conf --- apps/emqx_connector/etc/emqx_connector.conf | 16 ++++ apps/emqx_connector/src/emqx_connector.erl | 1 + .../emqx_connector/src/emqx_connector_app.erl | 17 +++++ apps/emqx_resource/src/emqx_resource.erl | 22 ++++-- .../src/emqx_resource_instance.erl | 75 ++++++++++++------- 5 files changed, 96 insertions(+), 35 deletions(-) create mode 100644 apps/emqx_connector/src/emqx_connector.erl diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf index 54b57ebe6..d93db8c34 100644 --- a/apps/emqx_connector/etc/emqx_connector.conf +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -1,3 +1,19 @@ ##-------------------------------------------------------------------- ## EMQ X CONNECTOR Plugin ##-------------------------------------------------------------------- + +## Base directory for emqx_connector indicating where to load configs from disk. +## +## Value: String +## Default: "{{ etc_dir }}/connectors/" +emqx_connectors: [ + {id: "mysql-abc", + type: mysql_connenctor, + config: {} + }, + {id: "pgsql-123", + type: pgsql_connenctor, + config: {} + }, +] + diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl new file mode 100644 index 000000000..ccb88c81a --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -0,0 +1 @@ +-module(emqx_connector). diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index 402e94187..0e64d11bc 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -12,9 +12,26 @@ -export([start/2, stop/1]). start(_StartType, _StartArgs) -> + load_config(), emqx_connector_sup:start_link(). stop(_State) -> ok. %% internal functions + +load_config() -> + case hocon:load("etc/plugins/emqx_connector.conf", #{format => map}) of + {ok, #{<<"emqx_connectors">> := Connectors}} -> + lists:foreach(fun load_connector/1, Connectors); + {error, Reason} -> + error(Reason) + end. + +load_connector(Config) -> + case emqx_resource:load_instance_from_config(Config) of + {ok, _} -> ok; + {error, already_created} -> ok; + {error, Reason} -> + error({load_connector, Reason}) + end. diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b9107d328..00ea1f868 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -71,7 +71,9 @@ , list_instances_verbose/0 %% list all the instances , get_instance/1 %% return the data of the instance , get_instance_by_type/1 %% return all the instances of the same resource type - , load_instances/1 %% load instances from config files + , load_instances_from_dir/1 %% load instances from a directory + , load_instance_from_file/1 %% load an instance from a config file + , load_instance_from_config/1 %% load an instance from a map or json-string config % , dependents/1 % , inc_counter/2 %% increment the counter of the instance % , inc_counter/3 %% increment the counter by a given integer @@ -208,9 +210,17 @@ list_instances_verbose() -> get_instance_by_type(ResourceType) -> emqx_resource_instance:lookup_by_type(ResourceType). --spec load_instances(Dir :: string()) -> ok. -load_instances(Dir) -> - emqx_resource_instance:load(Dir). +-spec load_instances_from_dir(Dir :: string()) -> ok. +load_instances_from_dir(Dir) -> + emqx_resource_instance:load_dir(Dir). + +-spec load_instance_from_file(File :: string()) -> ok. +load_instance_from_file(File) -> + emqx_resource_instance:load_file(File). + +-spec load_instance_from_config(binary() | map()) -> ok. +load_instance_from_config(Config) -> + emqx_resource_instance:load_config(Config). -spec call_start(instance_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. @@ -240,7 +250,7 @@ parse_config(ResourceType, RawConfig) when is_binary(RawConfig) -> Error -> Error end; parse_config(ResourceType, RawConfigTerm) -> - parse_config(ResourceType, jsx:encode(#{<<"config">> => RawConfigTerm})). + parse_config(ResourceType, jsx:encode(#{config => RawConfigTerm})). -spec do_parse_config(resource_type(), map()) -> {ok, resource_config()} | {error, term()}. do_parse_config(ResourceType, MapConfig) -> @@ -261,7 +271,7 @@ resource_type_from_str(ResourceType) -> false -> {error, {invalid_resource, Mod}} end catch error:badarg -> - {error, {not_found, ResourceType}} + {error, {resourec_not_found, ResourceType}} end. call_instance(InstId, Query) -> diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index ca5e4829e..3a750ebf3 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -23,10 +23,13 @@ -export([start_link/2]). %% load resource instances from *.conf files --export([ load/1 +-export([ load_dir/1 + , load_file/1 + , load_config/1 , lookup/1 , list_all/0 , lookup_by_type/1 + , create_local/3 ]). -export([ hash_call/2 @@ -82,8 +85,8 @@ lookup_by_type(ResourceType) -> [Data || #{mod := Mod} = Data <- list_all() , Mod =:= ResourceType]. --spec load(Dir :: string()) -> ok. -load(Dir) -> +-spec load_dir(Dir :: string()) -> ok. +load_dir(Dir) -> lists:foreach(fun load_file/1, filelib:wildcard(filename:join([Dir, "*.conf"]))). load_file(File) -> @@ -91,40 +94,48 @@ load_file(File) -> {error, Reason} -> logger:error("load resource from ~p failed: ~p", [File, Reason]); RawConfig -> - case hocon:binary(RawConfig, #{format => map}) of - {ok, #{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr, - <<"config">> := MapConfig}} -> - case emqx_resource:resource_type_from_str(ResourceTypeStr) of - {ok, ResourceType} -> - parse_and_load_config(Id, ResourceType, MapConfig); - {error, Reason} -> - logger:error("no such resource type: ~s, ~p", - [ResourceTypeStr, Reason]) - end; + case load_config(RawConfig) of + {ok, Data} -> + logger:debug("loaded resource instance from file: ~p, data: ~p", + [File, Data]); {error, Reason} -> logger:error("load resource from ~p failed: ~p", [File, Reason]) end end. -parse_and_load_config(InstId, ResourceType, MapConfig) -> - case emqx_resource:parse_config(ResourceType, MapConfig) of - {error, Reason} -> - logger:error("parse config for resource ~p of type ~p failed: ~p", - [InstId, ResourceType, Reason]); - {ok, InstConf} -> - create_instance_local(InstId, ResourceType, InstConf) +-spec load_config(binary() | map()) -> {ok, resource_data()} | {error, term()}. +load_config(RawConfig) when is_binary(RawConfig) -> + case hocon:binary(RawConfig, #{format => map}) of + {ok, ConfigTerm} -> load_config(ConfigTerm); + Error -> Error + end; + +load_config(#{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr, + <<"config">> := MapConfig}) -> + case emqx_resource:resource_type_from_str(ResourceTypeStr) of + {ok, ResourceType} -> parse_and_load_config(Id, ResourceType, MapConfig); + Error -> Error end. -create_instance_local(InstId, ResourceType, InstConf) -> - case do_create(InstId, ResourceType, InstConf) of - {ok, Data} -> - logger:debug("created ~p resource instance: ~p from config: ~p, Data: ~p", - [ResourceType, InstId, InstConf, Data]); - {error, Reason} -> - logger:error("create ~p resource instance: ~p failed: ~p, config: ~p", - [ResourceType, InstId, Reason, InstConf]) +parse_and_load_config(InstId, ResourceType, MapConfig) -> + case emqx_resource:parse_config(ResourceType, MapConfig) of + {ok, InstConf} -> create_local(InstId, ResourceType, InstConf); + Error -> Error end. +create_local(InstId, ResourceType, InstConf) -> + case hash_call(InstId, {create, InstId, ResourceType, InstConf}, 15000) of + {ok, Data} -> {ok, Data}; + Error -> Error + end. + +save_config_to_disk(InstId, ResourceType, Config) -> + file:write_file(filename:join([emqx_data_dir(), binary_to_list(InstId) ++ ".conf"]), + jsx:encode(#{id => InstId, resource_type => ResourceType, config => Config})). + +emqx_data_dir() -> + "data". + %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ @@ -205,7 +216,13 @@ do_create(InstId, ResourceType, Config) -> #{mod => ResourceType, config => Config, state => ResourceState, status => stopped}}), _ = do_health_check(InstId), - {ok, force_lookup(InstId)}; + case save_config_to_disk(InstId, ResourceType, Config) of + ok -> {ok, force_lookup(InstId)}; + {error, Reason} -> + logger:error("save config for ~p resource ~p to disk failed: ~p", + [ResourceType, InstId, Reason]), + {error, Reason} + end; {error, Reason} -> logger:error("start ~s resource ~s failed: ~p", [ResourceType, InstId, Reason]), {error, Reason} From 5d52ce044d1a6cd9288375a8360f1d410411f1cf Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 2 Jun 2021 01:58:26 +0800 Subject: [PATCH 04/19] feat(emqx_resource): read and save configs from and to file --- .../data/app.2021.06.01.22.32.03.config | 4 +++ .../data/vm.2021.06.01.22.32.03.args | 0 apps/emqx_connector/etc/emqx_connector.conf | 26 +++++++++---------- .../emqx_connector/priv/emqx_connector.schema | 5 ++++ .../emqx_connector/src/emqx_connector_app.erl | 2 +- .../src/emqx_connector_mysql.erl | 10 +++++-- .../src/emqx_connector_schema_lib.erl | 6 ++++- apps/emqx_resource/include/emqx_resource.hrl | 2 +- apps/emqx_resource/src/emqx_resource.app.src | 3 ++- apps/emqx_resource/src/emqx_resource.erl | 10 ++++++- apps/emqx_resource/src/emqx_resource_api.erl | 2 +- .../src/emqx_resource_instance.erl | 9 ++++--- 12 files changed, 55 insertions(+), 24 deletions(-) create mode 100644 apps/emqx_connector/data/app.2021.06.01.22.32.03.config create mode 100644 apps/emqx_connector/data/vm.2021.06.01.22.32.03.args diff --git a/apps/emqx_connector/data/app.2021.06.01.22.32.03.config b/apps/emqx_connector/data/app.2021.06.01.22.32.03.config new file mode 100644 index 000000000..29e2a1f7a --- /dev/null +++ b/apps/emqx_connector/data/app.2021.06.01.22.32.03.config @@ -0,0 +1,4 @@ +[{connectors,[#{<<"id">> => <<"mysql-abc">>, + <<"type">> => <<"mysql_connenctor">>}, + #{<<"id">> => <<"pgsql-123">>, + <<"type">> => <<"pgsql_connenctor">>}]}]. diff --git a/apps/emqx_connector/data/vm.2021.06.01.22.32.03.args b/apps/emqx_connector/data/vm.2021.06.01.22.32.03.args new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf index d93db8c34..06f25cde4 100644 --- a/apps/emqx_connector/etc/emqx_connector.conf +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -2,18 +2,18 @@ ## EMQ X CONNECTOR Plugin ##-------------------------------------------------------------------- -## Base directory for emqx_connector indicating where to load configs from disk. -## -## Value: String -## Default: "{{ etc_dir }}/connectors/" -emqx_connectors: [ - {id: "mysql-abc", - type: mysql_connenctor, - config: {} - }, - {id: "pgsql-123", - type: pgsql_connenctor, - config: {} - }, +connectors: [ + {id: "mysql-abc" + resource_type: emqx_connector_mysql + config: { + server: "127.0.0.1:3306" + database: mqtt + pool_size: 1 + user: root + password: public + auto_reconnect: true + ssl: false + } + } ] diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema index b8476c4d9..cdc079973 100644 --- a/apps/emqx_connector/priv/emqx_connector.schema +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -1,2 +1,7 @@ %%-*- mode: erlang -*- %% emqx_connector config mapping + +{mapping, "connectors", "connectors", [ + {default, []}, + {datatype, string} +]}. \ No newline at end of file diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index 0e64d11bc..f554cc117 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -22,7 +22,7 @@ stop(_State) -> load_config() -> case hocon:load("etc/plugins/emqx_connector.conf", #{format => map}) of - {ok, #{<<"emqx_connectors">> := Connectors}} -> + {ok, #{<<"connectors">> := Connectors}} -> lists:foreach(fun load_connector/1, Connectors); {error, Reason} -> error(Reason) diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index ad9b9a4d1..c34d361ce 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -5,7 +5,9 @@ -emqx_resource_api_path("connectors/mysql"). --export([fields/1]). +-export([ fields/1 + , on_config_to_file/1 + ]). %% callbacks of behaviour emqx_resource -export([ on_start/2 @@ -18,10 +20,14 @@ -export([do_health_check/1]). +%%===================================================================== fields("config") -> emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). +on_config_to_file(#{server := Server} = Config) -> + Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}. + %% =================================================================== on_start(InstId, #{server := {Host, Port}, @@ -58,7 +64,7 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL]}, no_handover) of {error, Reason} -> logger:debug("mysql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), - emqx_resource:query_failure(AfterQuery); + emqx_resource:query_failed(AfterQuery); _ -> emqx_resource:query_success(AfterQuery) end, diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 8e69245b9..341901750 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -6,6 +6,7 @@ ]). -export([ to_ip_port/1 + , ip_port_to_string/1 ]). -typerefl_from_string({ip_port/0, emqx_connector_schema_lib, to_ip_port}). @@ -99,4 +100,7 @@ to_ip_port(Str) -> _ -> {error, Str} end; _ -> {error, Str} - end. \ No newline at end of file + end. + +ip_port_to_string({Ip, Port}) -> + inet:ntoa(Ip) ++ ":" ++ integer_to_list(Port). diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 1f75b453e..123854bc9 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -type resource_type() :: module(). -type instance_id() :: binary(). --type resource_config() :: jsx:json_term(). +-type resource_config() :: term(). -type resource_spec() :: map(). -type resource_state() :: term(). -type resource_data() :: #{ diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index af9f48cc6..13330b061 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -7,7 +7,8 @@ [kernel, stdlib, gproc, - hocon + hocon, + jsx ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 00ea1f868..b5c1ce730 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -65,6 +65,7 @@ , call_health_check/3 %% verify if the resource is working normally , call_stop/3 %% stop the instance , call_config_merge/4 %% merge the config when updating + , call_config_to_file/2 ]). -export([ list_instances/0 %% list all the instances, id only. @@ -85,12 +86,15 @@ , on_health_check/2 , on_api_reply_format/1 , on_config_merge/3 + , on_config_to_file/1 ]). -callback on_api_reply_format(resource_data()) -> map(). -callback on_config_merge(resource_config(), resource_config(), term()) -> resource_config(). +-callback on_config_to_file(resource_config()) -> jsx:json_term(). + %% when calling emqx_resource:start/1 -callback on_start(instance_id(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. @@ -241,6 +245,10 @@ call_stop(InstId, Mod, ResourceState) -> call_config_merge(Mod, OldConfig, NewConfig, Params) -> ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)). +-spec call_config_to_file(module(), resource_config()) -> jsx:json_term(). +call_config_to_file(Mod, Config) -> + ?SAFE_CALL(Mod:on_config_to_file(Config)). + -spec parse_config(resource_type(), binary() | term()) -> {ok, resource_config()} | {error, term()}. parse_config(ResourceType, RawConfig) when is_binary(RawConfig) -> @@ -271,7 +279,7 @@ resource_type_from_str(ResourceType) -> false -> {error, {invalid_resource, Mod}} end catch error:badarg -> - {error, {resourec_not_found, ResourceType}} + {error, {resource_not_found, ResourceType}} end. call_instance(InstId, Query) -> diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl index cc32d8a11..61d162ee6 100644 --- a/apps/emqx_resource/src/emqx_resource_api.erl +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -34,7 +34,7 @@ get(Mod, #{id := Id}, _Params) -> put(Mod, #{id := Id}, Params) -> ConfigParams = proplists:get_value(<<"config">>, Params), - ResourceTypeStr = proplists:get_value(<<"resource_type">>, Params), + ResourceTypeStr = proplists:get_value(<<"resource_type">>, Params, #{}), case emqx_resource:resource_type_from_str(ResourceTypeStr) of {ok, ResourceType} -> do_put(Mod, stringnify(Id), ConfigParams, ResourceType, Params); diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 3a750ebf3..ff7158c9c 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -110,8 +110,8 @@ load_config(RawConfig) when is_binary(RawConfig) -> Error -> Error end; -load_config(#{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr, - <<"config">> := MapConfig}) -> +load_config(#{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr} = Config) -> + MapConfig = maps:get(<<"config">>, Config, #{}), case emqx_resource:resource_type_from_str(ResourceTypeStr) of {ok, ResourceType} -> parse_and_load_config(Id, ResourceType, MapConfig); Error -> Error @@ -130,8 +130,11 @@ create_local(InstId, ResourceType, InstConf) -> end. save_config_to_disk(InstId, ResourceType, Config) -> + %% TODO: send an event to the config handler, and the hander (single process) + %% will dump configs for all instances (from an ETS table) to a file. file:write_file(filename:join([emqx_data_dir(), binary_to_list(InstId) ++ ".conf"]), - jsx:encode(#{id => InstId, resource_type => ResourceType, config => Config})). + jsx:encode(#{id => InstId, resource_type => ResourceType, + config => emqx_resource:call_config_to_file(Config)})). emqx_data_dir() -> "data". From 49c5edce2e67939485c5fc933b549b2c5e89e6d3 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 2 Jun 2021 16:28:52 +0800 Subject: [PATCH 05/19] fix(emqx_connector): start emqx_connector_mysql failed --- apps/emqx_connector/priv/emqx_connector.schema | 2 +- apps/emqx_connector/src/emqx_connector.app.src | 3 ++- apps/emqx_connector/src/emqx_connector_app.erl | 2 ++ apps/emqx_resource/src/emqx_resource_instance.erl | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema index cdc079973..8d81635d5 100644 --- a/apps/emqx_connector/priv/emqx_connector.schema +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -1,7 +1,7 @@ %%-*- mode: erlang -*- %% emqx_connector config mapping -{mapping, "connectors", "connectors", [ +{mapping, "connectors", "emqx_connector.connectors", [ {default, []}, {datatype, string} ]}. \ No newline at end of file diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 16a1d0869..a821b8f13 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -6,7 +6,8 @@ {applications, [kernel, stdlib, - emqx_resource + emqx_resource, + ecpool ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index f554cc117..0ba33e79b 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -11,6 +11,8 @@ -export([start/2, stop/1]). +-export([load_config/0]). + start(_StartType, _StartArgs) -> load_config(), emqx_connector_sup:start_link(). diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index ff7158c9c..60fdea74b 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -134,7 +134,7 @@ save_config_to_disk(InstId, ResourceType, Config) -> %% will dump configs for all instances (from an ETS table) to a file. file:write_file(filename:join([emqx_data_dir(), binary_to_list(InstId) ++ ".conf"]), jsx:encode(#{id => InstId, resource_type => ResourceType, - config => emqx_resource:call_config_to_file(Config)})). + config => emqx_resource:call_config_to_file(ResourceType, Config)})). emqx_data_dir() -> "data". From ebe1228b2c0774252c8d1150e56b7ccca3c07a31 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 2 Jun 2021 18:41:48 +0800 Subject: [PATCH 06/19] fix(emqx_connector): don't return error when stopping a non-existing pool --- apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index 3db29b427..dddc44f4e 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -39,6 +39,7 @@ start_pool(Name, Mod, Options) -> stop_pool(Name) -> case ecpool:stop_sup_pool(Name) of ok -> logger:log(info, "Destroyed ~0p Successfully", [Name]); + {error, not_found} -> ok; {error, Reason} -> logger:log(error, "Destroy ~0p failed, ~0p", [Name, Reason]), error({stop_pool_failed, Name}) From bea5966ac8926bb1d862c3d070020b4b3beb7ceb Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 2 Jun 2021 19:10:24 +0800 Subject: [PATCH 07/19] fix(emqx_connector): remove unrelated files and add license headers --- apps/emqx_connector/LICENSE | 191 ------------------ apps/emqx_connector/README.md | 26 ++- .../data/app.2021.06.01.22.32.03.config | 4 - .../data/vm.2021.06.01.22.32.03.args | 0 apps/emqx_connector/src/emqx_connector.erl | 39 ++++ .../emqx_connector/src/emqx_connector_app.erl | 40 ++-- .../src/emqx_connector_mysql.erl | 15 ++ .../src/emqx_connector_schema_lib.erl | 15 ++ .../emqx_connector/src/emqx_connector_sup.erl | 29 +-- 9 files changed, 121 insertions(+), 238 deletions(-) delete mode 100644 apps/emqx_connector/LICENSE delete mode 100644 apps/emqx_connector/data/app.2021.06.01.22.32.03.config delete mode 100644 apps/emqx_connector/data/vm.2021.06.01.22.32.03.args diff --git a/apps/emqx_connector/LICENSE b/apps/emqx_connector/LICENSE deleted file mode 100644 index 2f97fdd03..000000000 --- a/apps/emqx_connector/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2021, Shawn <506895667@qq.com>. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md index cf64b5b68..845da77d0 100644 --- a/apps/emqx_connector/README.md +++ b/apps/emqx_connector/README.md @@ -1,9 +1,23 @@ -emqx_connector -===== +# emqx_connector -An OTP application +A connector is an object that maintains the data related to external resources. -Build ------ +For example, a mysql connector is an object that maintains all the mysql connection +related parameters (configs) and the TCP connections to the mysql server. - $ rebar3 compile +An mysql connector can be used as following: + +``` +(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). +[#{config => + #{auto_reconnect => true,cacertfile => [],certfile => [], + database => "mqtt",keyfile => [],password => "public", + pool_size => 1, + server => {{127,0,0,1},3306}, + ssl => false,user => "root",verify => false}, + id => <<"mysql-abc">>,mod => emqx_connector_mysql, + state => #{poolname => 'mysql-abc'}, + status => started}] +(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). +{ok,[<<"count(1)">>],[[1]]} +``` diff --git a/apps/emqx_connector/data/app.2021.06.01.22.32.03.config b/apps/emqx_connector/data/app.2021.06.01.22.32.03.config deleted file mode 100644 index 29e2a1f7a..000000000 --- a/apps/emqx_connector/data/app.2021.06.01.22.32.03.config +++ /dev/null @@ -1,4 +0,0 @@ -[{connectors,[#{<<"id">> => <<"mysql-abc">>, - <<"type">> => <<"mysql_connenctor">>}, - #{<<"id">> => <<"pgsql-123">>, - <<"type">> => <<"pgsql_connenctor">>}]}]. diff --git a/apps/emqx_connector/data/vm.2021.06.01.22.32.03.args b/apps/emqx_connector/data/vm.2021.06.01.22.32.03.args deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl index ccb88c81a..c18106950 100644 --- a/apps/emqx_connector/src/emqx_connector.erl +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -1 +1,40 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_connector). + +-export([ load_from_config/1 + , load_connectors/1 + , load_connector/1 + ]). + +load_from_config(Filename) -> + case hocon:load(Filename, #{format => map}) of + {ok, #{<<"connectors">> := Connectors}} -> + load_connectors(Connectors); + {error, Reason} -> + error(Reason) + end. + +load_connectors(Connectors) -> + lists:foreach(fun load_connector/1, Connectors). + +load_connector(Config) -> + case emqx_resource:load_instance_from_config(Config) of + {ok, _} -> ok; + {error, already_created} -> ok; + {error, Reason} -> + error({load_connector, Reason}) + end. diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index 0ba33e79b..af652733f 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -1,7 +1,18 @@ -%%%------------------------------------------------------------------- -%% @doc emqx_connector public API -%% @end -%%%------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_connector_app). @@ -11,29 +22,12 @@ -export([start/2, stop/1]). --export([load_config/0]). - start(_StartType, _StartArgs) -> - load_config(), + Connectors = proplists:get_value(connectors, application:get_all_env(emqx_connector), []), + emqx_connector:load_connectors(Connectors), emqx_connector_sup:start_link(). stop(_State) -> ok. %% internal functions - -load_config() -> - case hocon:load("etc/plugins/emqx_connector.conf", #{format => map}) of - {ok, #{<<"connectors">> := Connectors}} -> - lists:foreach(fun load_connector/1, Connectors); - {error, Reason} -> - error(Reason) - end. - -load_connector(Config) -> - case emqx_resource:load_instance_from_config(Config) of - {ok, _} -> ok; - {error, already_created} -> ok; - {error, Reason} -> - error({load_connector, Reason}) - end. diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index c34d361ce..4dd43de4d 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_connector_mysql). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 341901750..9724c1db6 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_connector_schema_lib). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index 454dd7084..603b9a8ad 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -1,8 +1,18 @@ -%%%------------------------------------------------------------------- -%% @doc emqx_connector top level supervisor. -%% @end -%%%------------------------------------------------------------------- - +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_connector_sup). -behaviour(supervisor). @@ -16,15 +26,6 @@ start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). -%% sup_flags() = #{strategy => strategy(), % optional -%% intensity => non_neg_integer(), % optional -%% period => pos_integer()} % optional -%% child_spec() = #{id => child_id(), % mandatory -%% start => mfargs(), % mandatory -%% restart => restart(), % optional -%% shutdown => shutdown(), % optional -%% type => worker(), % optional -%% modules => modules()} % optional init([]) -> SupFlags = #{strategy => one_for_all, intensity => 0, From 3da62e59d6cad94772a6488dd7823eae5eac921e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 3 Jun 2021 01:24:30 +0800 Subject: [PATCH 08/19] fix(emqx_connector): the HTTP API for emqx_connector_mysql --- .../src/emqx_connector_mysql.erl | 20 +++++++++-- .../src/emqx_connector_schema_lib.erl | 2 +- apps/emqx_resource/rebar.config | 2 +- apps/emqx_resource/src/emqx_resource.erl | 36 ++++++++++++++----- apps/emqx_resource/src/emqx_resource_api.erl | 15 ++++---- .../src/emqx_resource_instance.erl | 2 +- .../src/emqx_resource_transform.erl | 16 ++++----- 7 files changed, 62 insertions(+), 31 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 4dd43de4d..3d22b4c99 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -21,7 +21,8 @@ -emqx_resource_api_path("connectors/mysql"). -export([ fields/1 - , on_config_to_file/1 + , on_jsonify/1 + , on_api_reply_format/1 ]). %% callbacks of behaviour emqx_resource @@ -40,8 +41,21 @@ fields("config") -> emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). -on_config_to_file(#{server := Server} = Config) -> - Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}. +on_jsonify(#{server := Server, user := User, database := DB, password := Passwd, + cacertfile := CAFile, keyfile := KeyFile, certfile := CertFile} = Config) -> + Config#{ + user => list_to_binary(User), + database => list_to_binary(DB), + password => list_to_binary(Passwd), + server => emqx_connector_schema_lib:ip_port_to_string(Server), + cacertfile => list_to_binary(CAFile), + keyfile => list_to_binary(KeyFile), + certfile => list_to_binary(CertFile) + }. + +on_api_reply_format(ResourceData) -> + #{config := Conf} = Reply0 = emqx_resource_api:default_api_reply_format(ResourceData), + Reply0#{config => maps:without([cacertfile, keyfile, certfile, verify], Conf)}. %% =================================================================== diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 9724c1db6..39bb72113 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -118,4 +118,4 @@ to_ip_port(Str) -> end. ip_port_to_string({Ip, Port}) -> - inet:ntoa(Ip) ++ ":" ++ integer_to_list(Port). + iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]). diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 148af92bd..305312b4e 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -1,6 +1,6 @@ {erl_opts, [ debug_info , nowarn_unused_import - %, {d, 'RESOURCE_DEBUG'} + , {d, 'RESOURCE_DEBUG'} ]}. {erl_first_files, ["src/emqx_resource_transform.erl"]}. diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b5c1ce730..c432e683e 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -65,7 +65,8 @@ , call_health_check/3 %% verify if the resource is working normally , call_stop/3 %% stop the instance , call_config_merge/4 %% merge the config when updating - , call_config_to_file/2 + , call_jsonify/2 + , call_api_reply_format/2 ]). -export([ list_instances/0 %% list all the instances, id only. @@ -84,16 +85,16 @@ -optional_callbacks([ on_query/4 , on_health_check/2 - , on_api_reply_format/1 , on_config_merge/3 - , on_config_to_file/1 + , on_jsonify/1 + , on_api_reply_format/1 ]). --callback on_api_reply_format(resource_data()) -> map(). +-callback on_api_reply_format(resource_data()) -> jsx:json_term(). -callback on_config_merge(resource_config(), resource_config(), term()) -> resource_config(). --callback on_config_to_file(resource_config()) -> jsx:json_term(). +-callback on_jsonify(resource_config()) -> jsx:json_term(). %% when calling emqx_resource:start/1 -callback on_start(instance_id(), resource_config()) -> @@ -243,11 +244,28 @@ call_stop(InstId, Mod, ResourceState) -> -spec call_config_merge(module(), resource_config(), resource_config(), term()) -> resource_config(). call_config_merge(Mod, OldConfig, NewConfig, Params) -> - ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)). + case erlang:function_exported(Mod, on_jsonify, 1) of + true -> + ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)); + false when is_map(OldConfig), is_map(NewConfig) -> + maps:merge(OldConfig, NewConfig); + false -> + NewConfig + end. --spec call_config_to_file(module(), resource_config()) -> jsx:json_term(). -call_config_to_file(Mod, Config) -> - ?SAFE_CALL(Mod:on_config_to_file(Config)). +-spec call_jsonify(module(), resource_config()) -> jsx:json_term(). +call_jsonify(Mod, Config) -> + case erlang:function_exported(Mod, on_jsonify, 1) of + false -> Config; + true -> ?SAFE_CALL(Mod:on_jsonify(Config)) + end. + +-spec call_api_reply_format(module(), resource_data()) -> jsx:json_term(). +call_api_reply_format(Mod, Data) -> + case erlang:function_exported(Mod, on_api_reply_format, 1) of + false -> emqx_resource_api:default_api_reply_format(Data); + true -> ?SAFE_CALL(Mod:on_api_reply_format(Data)) + end. -spec parse_config(resource_type(), binary() | term()) -> {ok, resource_config()} | {error, term()}. diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl index 61d162ee6..1e19cdede 100644 --- a/apps/emqx_resource/src/emqx_resource_api.erl +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -20,6 +20,9 @@ , put/3 , delete/3 ]). + +-export([default_api_reply_format/1]). + get_all(Mod, _Binding, _Params) -> {200, #{code => 0, data => [format_data(Mod, Data) || Data <- emqx_resource:list_instances_verbose()]}}. @@ -63,15 +66,11 @@ delete(_Mod, #{id := Id}, _Params) -> end. format_data(Mod, Data) -> - case erlang:function_exported(Mod, on_api_reply_format, 1) of - false -> - default_api_reply_format(Data); - true -> - Mod:on_api_reply_format(Data) - end. + emqx_resource:call_api_reply_format(Mod, Data). -default_api_reply_format(#{id := Id, status := Status, config := Config}) -> - #{node => node(), id => Id, status => Status, config => Config}. +default_api_reply_format(#{id := Id, mod := Mod, status := Status, config := Config}) -> + #{node => node(), id => Id, status => Status, resource_type => Mod, + config => emqx_resource:call_jsonify(Mod, Config)}. stringnify(Bin) when is_binary(Bin) -> Bin; stringnify(Str) when is_list(Str) -> list_to_binary(Str); diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 60fdea74b..dbbf052a1 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -134,7 +134,7 @@ save_config_to_disk(InstId, ResourceType, Config) -> %% will dump configs for all instances (from an ETS table) to a file. file:write_file(filename:join([emqx_data_dir(), binary_to_list(InstId) ++ ".conf"]), jsx:encode(#{id => InstId, resource_type => ResourceType, - config => emqx_resource:call_config_to_file(ResourceType, Config)})). + config => emqx_resource:call_jsonify(ResourceType, Config)})). emqx_data_dir() -> "data". diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl index 23e14013c..cd6c7e4ae 100644 --- a/apps/emqx_resource/src/emqx_resource_transform.erl +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -57,7 +57,8 @@ forms(_, []) -> []. form(Mod, Form) -> case Form of ?Q("-emqx_resource_api_path('@Path').") -> - {fix_spec_attrs() ++ fix_api_attrs(erl_syntax:concrete(Path)) ++ fix_api_exports(), + {fix_spec_attrs() ++ fix_api_attrs(Mod, erl_syntax:concrete(Path)) + ++ fix_api_exports(), [], fix_spec_funcs(Mod) ++ fix_api_funcs(Mod)}; _ -> @@ -75,18 +76,17 @@ fix_spec_funcs(_Mod) -> , ?Q("structs() -> [\"config\"].") ]. -fix_api_attrs(Path0) -> - BaseName = filename:basename(Path0), - Path = "/" ++ BaseName, +fix_api_attrs(Mod, Path) -> + BaseName = atom_to_list(Mod), [erl_syntax:revert( erl_syntax:attribute(?Q("rest_api"), [ erl_syntax:abstract(#{ - name => list_to_atom(Name ++ "_log_tracers"), + name => list_to_atom(Act ++ "_" ++ BaseName), method => Method, path => mk_path(Path, WithId), func => Func, - descr => Name ++ " the " ++ BaseName})])) - || {Name, Method, WithId, Func} <- [ + descr => Act ++ " the " ++ BaseName})])) + || {Act, Method, WithId, Func} <- [ {"list", 'GET', noid, api_get_all}, {"get", 'GET', id, api_get}, {"update", 'PUT', id, api_put}, @@ -110,5 +110,5 @@ fix_api_funcs(Mod) -> emqx_resource_api:delete('@Mod@', Binding, Params).")) ]. -mk_path(Path, id) -> Path ++ "/:bin:id"; +mk_path(Path, id) -> string:trim(Path, trailing, "/") ++ "/:bin:id"; mk_path(Path, noid) -> Path. From f1552f4f4fd51c2a9b5966fd42be804cbe32da82 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 4 Jun 2021 23:47:16 +0800 Subject: [PATCH 09/19] feat(emqx_data_bridge): create emqx_data_bridge --- apps/emqx_connector/README.md | 8 ++- apps/emqx_connector/etc/emqx_connector.conf | 15 ----- .../emqx_connector/priv/emqx_connector.schema | 5 -- apps/emqx_connector/src/emqx_connector.erl | 24 ------- .../emqx_connector/src/emqx_connector_app.erl | 2 - .../src/emqx_connector_mysql.erl | 41 ++++++------ .../src/emqx_connector_schema_lib.erl | 12 +--- apps/emqx_data_bridge/.gitignore | 19 ++++++ apps/emqx_data_bridge/README.md | 10 +++ .../etc/emqx_data_bridge.conf | 30 +++++++++ .../priv/emqx_data_bridge.schema | 16 +++++ apps/emqx_data_bridge/rebar.config | 7 ++ .../src/emqx_data_bridge.app.src | 15 +++++ .../emqx_data_bridge/src/emqx_data_bridge.erl | 9 +++ .../src/emqx_data_bridge_app.erl | 32 ++++++++++ .../src/emqx_data_bridge_monitor.erl | 64 +++++++++++++++++++ .../src/emqx_data_bridge_sup.erl | 41 ++++++++++++ apps/emqx_resource/examples/log_tracer.erl | 6 +- .../examples/log_tracer_schema.erl | 7 +- apps/emqx_resource/src/emqx_resource.erl | 46 ++++++------- apps/emqx_resource/src/emqx_resource_api.erl | 2 +- .../src/emqx_resource_instance.erl | 45 +------------ .../src/emqx_resource_transform.erl | 10 ++- data/loaded_plugins.tmpl | 1 + rebar.config.erl | 2 + 25 files changed, 311 insertions(+), 158 deletions(-) create mode 100644 apps/emqx_data_bridge/.gitignore create mode 100644 apps/emqx_data_bridge/README.md create mode 100644 apps/emqx_data_bridge/etc/emqx_data_bridge.conf create mode 100644 apps/emqx_data_bridge/priv/emqx_data_bridge.schema create mode 100644 apps/emqx_data_bridge/rebar.config create mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge.app.src create mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge.erl create mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge_app.erl create mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl create mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md index 845da77d0..879669f93 100644 --- a/apps/emqx_connector/README.md +++ b/apps/emqx_connector/README.md @@ -1,8 +1,12 @@ # emqx_connector -A connector is an object that maintains the data related to external resources. +This application is a collection of `connectors`. -For example, a mysql connector is an object that maintains all the mysql connection +A `connector` is a callback module of `emqx_resource` that maintains the data related to +external resources. Put all resource related callback modules in a single application is good as +we can put some util functions/modules here for reusing purpose. + +For example, a mysql connector is an emqx resource that maintains all the mysql connection related parameters (configs) and the TCP connections to the mysql server. An mysql connector can be used as following: diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf index 06f25cde4..db4402d47 100644 --- a/apps/emqx_connector/etc/emqx_connector.conf +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -2,18 +2,3 @@ ## EMQ X CONNECTOR Plugin ##-------------------------------------------------------------------- -connectors: [ - {id: "mysql-abc" - resource_type: emqx_connector_mysql - config: { - server: "127.0.0.1:3306" - database: mqtt - pool_size: 1 - user: root - password: public - auto_reconnect: true - ssl: false - } - } -] - diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema index 8d81635d5..b8476c4d9 100644 --- a/apps/emqx_connector/priv/emqx_connector.schema +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -1,7 +1,2 @@ %%-*- mode: erlang -*- %% emqx_connector config mapping - -{mapping, "connectors", "emqx_connector.connectors", [ - {default, []}, - {datatype, string} -]}. \ No newline at end of file diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl index c18106950..dd0359348 100644 --- a/apps/emqx_connector/src/emqx_connector.erl +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -14,27 +14,3 @@ %% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_connector). - --export([ load_from_config/1 - , load_connectors/1 - , load_connector/1 - ]). - -load_from_config(Filename) -> - case hocon:load(Filename, #{format => map}) of - {ok, #{<<"connectors">> := Connectors}} -> - load_connectors(Connectors); - {error, Reason} -> - error(Reason) - end. - -load_connectors(Connectors) -> - lists:foreach(fun load_connector/1, Connectors). - -load_connector(Config) -> - case emqx_resource:load_instance_from_config(Config) of - {ok, _} -> ok; - {error, already_created} -> ok; - {error, Reason} -> - error({load_connector, Reason}) - end. diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index af652733f..4bbad75cf 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -23,8 +23,6 @@ -export([start/2, stop/1]). start(_StartType, _StartArgs) -> - Connectors = proplists:get_value(connectors, application:get_all_env(emqx_connector), []), - emqx_connector:load_connectors(Connectors), emqx_connector_sup:start_link(). stop(_State) -> diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 3d22b4c99..1bc1bbc80 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -20,8 +20,7 @@ -emqx_resource_api_path("connectors/mysql"). --export([ fields/1 - , on_jsonify/1 +-export([ on_jsonify/1 , on_api_reply_format/1 ]). @@ -37,36 +36,38 @@ -export([do_health_check/1]). %%===================================================================== -fields("config") -> +schema() -> emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). -on_jsonify(#{server := Server, user := User, database := DB, password := Passwd, - cacertfile := CAFile, keyfile := KeyFile, certfile := CertFile} = Config) -> +on_jsonify(#{<<"server">> := Server, <<"user">> := User, <<"database">> := DB, + <<"password">> := Passwd, <<"cacertfile">> := CAFile, + <<"keyfile">> := KeyFile, <<"certfile">> := CertFile} = Config) -> Config#{ - user => list_to_binary(User), - database => list_to_binary(DB), - password => list_to_binary(Passwd), - server => emqx_connector_schema_lib:ip_port_to_string(Server), - cacertfile => list_to_binary(CAFile), - keyfile => list_to_binary(KeyFile), - certfile => list_to_binary(CertFile) + <<"user">> => list_to_binary(User), + <<"database">> => list_to_binary(DB), + <<"password">> => list_to_binary(Passwd), + <<"server">> => emqx_connector_schema_lib:ip_port_to_string(Server), + <<"cacertfile">> => list_to_binary(CAFile), + <<"keyfile">> => list_to_binary(KeyFile), + <<"certfile">> => list_to_binary(CertFile) }. on_api_reply_format(ResourceData) -> #{config := Conf} = Reply0 = emqx_resource_api:default_api_reply_format(ResourceData), - Reply0#{config => maps:without([cacertfile, keyfile, certfile, verify], Conf)}. + Reply0#{config => maps:without([<<"cacertfile">>, <<"keyfile">>, + <<"certfile">>, <<"verify">>], Conf)}. %% =================================================================== -on_start(InstId, #{server := {Host, Port}, - database := DB, - user := User, - password := Password, - auto_reconnect := AutoReconn, - pool_size := PoolSize} = Config) -> +on_start(InstId, #{<<"server">> := {Host, Port}, + <<"database">> := DB, + <<"user">> := User, + <<"password">> := Password, + <<"auto_reconnect">> := AutoReconn, + <<"pool_size">> := PoolSize} = Config) -> logger:info("starting mysql connector: ~p, config: ~p", [InstId, Config]), - SslOpts = case maps:get(ssl, Config) of + SslOpts = case maps:get(<<"ssl">>, Config) of true -> [{ssl, [{server_name_indication, disable} | emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)]}]; diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 39bb72113..02bed956b 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -52,57 +52,47 @@ ssl_fields() -> , {verify, fun verify/1} ]. -server(mapping) -> "config.server"; server(type) -> ip_port(); server(validator) -> [?REQUIRED("the field 'server' is required")]; server(_) -> undefined. -database(mapping) -> "config.database"; database(type) -> string(); database(validator) -> [?REQUIRED("the field 'server' is required")]; database(_) -> undefined. -pool_size(mapping) -> "config.pool_size"; pool_size(type) -> integer(); pool_size(default) -> 8; pool_size(validator) -> [?MIN(1), ?MAX(64)]; pool_size(_) -> undefined. -user(mapping) -> "config.user"; user(type) -> string(); user(default) -> "root"; user(_) -> undefined. -password(mapping) -> "config.password"; password(type) -> string(); password(default) -> ""; password(_) -> undefined. -auto_reconnect(mapping) -> "config.auto_reconnect"; auto_reconnect(type) -> boolean(); auto_reconnect(default) -> true; auto_reconnect(_) -> undefined. -ssl(mapping) -> "config.ssl"; + ssl(type) -> boolean(); ssl(default) -> false; ssl(_) -> undefined. -cacertfile(mapping) -> "config.cacertfile"; cacertfile(type) -> string(); cacertfile(default) -> ""; cacertfile(_) -> undefined. -keyfile(mapping) -> "config.keyfile"; keyfile(type) -> string(); keyfile(default) -> ""; keyfile(_) -> undefined. -certfile(mapping) -> "config.certfile"; certfile(type) -> string(); certfile(default) -> ""; certfile(_) -> undefined. -verify(mapping) -> "config.verify"; verify(type) -> boolean(); verify(default) -> false; verify(_) -> undefined. diff --git a/apps/emqx_data_bridge/.gitignore b/apps/emqx_data_bridge/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_data_bridge/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_data_bridge/README.md b/apps/emqx_data_bridge/README.md new file mode 100644 index 000000000..8f76f17a5 --- /dev/null +++ b/apps/emqx_data_bridge/README.md @@ -0,0 +1,10 @@ +# emqx_data_bridge + +EMQ X Data Bridge is an application that managing the resources (see emqx_resource) used by emqx +rule engine. + +It provides CRUD HTTP APIs of the resources, and is also responsible for loading the resources at +startup, and saving configs of resources to `data/` after configs updated. + +The application depends on `emqx_connector` as that's where all the callback modules of `connector` +resources placed. diff --git a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf new file mode 100644 index 000000000..295173e6c --- /dev/null +++ b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf @@ -0,0 +1,30 @@ +##-------------------------------------------------------------------- +## EMQ X Bridge Plugin +##-------------------------------------------------------------------- + +emqx_data_bridge.bridges: [ + {name: "mysql-abc" + type: mysql + config: { + server: "127.0.0.1:3306" + database: mqtt + pool_size: 1 + user: root + password: public + auto_reconnect: true + ssl: false + } + }, + {name: "mysql-def" + type: mysql + config: { + server: "127.0.0.1:3306" + database: mqtt + pool_size: 1 + user: root + password: public + auto_reconnect: true + ssl: false + } + } +] diff --git a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema new file mode 100644 index 000000000..c9cb3f2c0 --- /dev/null +++ b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema @@ -0,0 +1,16 @@ +%%-*- mode: erlang -*- +%% emqx_data_bridge config mapping + +{mapping, "emqx_data_bridge.bridges", "emqx_data_bridge.bridges", [ + {default, []}, + {datatype, string} +]}. + +% fields("emqx_data_bridge") -> +% [ +% {bridges, +% [fun(mapping) -> "emqx_data_bridge.bridges"; +% (type) -> list(); +% (_) -> undefined +% end]} +% ] \ No newline at end of file diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_data_bridge/rebar.config new file mode 100644 index 000000000..cf4cfcf1b --- /dev/null +++ b/apps/emqx_data_bridge/rebar.config @@ -0,0 +1,7 @@ +{erl_opts, [debug_info]}. +{deps, []}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_data_bridge]} +]}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src new file mode 100644 index 000000000..360511d9b --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src @@ -0,0 +1,15 @@ +{application, emqx_data_bridge, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_data_bridge_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl new file mode 100644 index 000000000..cd26b9d3e --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -0,0 +1,9 @@ +-module(emqx_data_bridge). + +-export([ load_bridges/0 + ]). + +load_bridges() -> + Bridges = proplists:get_value(bridges, + application:get_all_env(emqx_data_bridge), []), + emqx_data_bridge_monitor:ensure_all_started(Bridges). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl new file mode 100644 index 000000000..4ff96e34e --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl @@ -0,0 +1,32 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_data_bridge_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_data_bridge_sup:start_link(), + ok = emqx_data_bridge:load_bridges(), + {ok, Sup}. + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl new file mode 100644 index 000000000..22c216038 --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl @@ -0,0 +1,64 @@ +%% This process monitors all the data bridges, and try to restart a bridge +%% when one of it stopped. +-module(emqx_data_bridge_monitor). + +-behaviour(gen_server). + +%% API functions +-export([ start_link/0 + , ensure_all_started/1 + ]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-record(state, {}). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +ensure_all_started(Configs) -> + gen_server:cast(?MODULE, {start_and_monitor, Configs}). + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast({start_and_monitor, Configs}, State) -> + ok = load_bridges(Configs), + {noreply, State}; + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%============================================================================ +load_bridges(Configs) -> + lists:foreach(fun load_bridge/1, Configs). + +load_bridge(#{<<"name">> := Name, <<"type">> := Type, + <<"config">> := Config}) -> + case emqx_resource:check_and_load_instance(Name, resource_type(Type), Config) of + {ok, _} -> ok; + {error, already_created} -> ok; + {error, Reason} -> + error({load_bridge, Reason}) + end. + +resource_type(<<"mysql">>) -> emqx_connector_mysql. \ No newline at end of file diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl new file mode 100644 index 000000000..516d9b5ab --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl @@ -0,0 +1,41 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_data_bridge_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 10}, + ChildSpecs = [ + #{id => emqx_data_bridge_monitor, + start => {emqx_data_bridge_monitor, start_link, []}, + restart => permanent, + type => worker, + modules => [emqx_data_bridge_monitor]}], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/apps/emqx_resource/examples/log_tracer.erl b/apps/emqx_resource/examples/log_tracer.erl index 99fc5bd3b..f4b531449 100644 --- a/apps/emqx_resource/examples/log_tracer.erl +++ b/apps/emqx_resource/examples/log_tracer.erl @@ -14,10 +14,10 @@ ]). %% callbacks for emqx_resource config schema --export([fields/1]). +-export([schema/0]). -fields(ConfPath) -> - log_tracer_schema:fields(ConfPath). +schema() -> + log_tracer_schema:schema(). on_start(InstId, Config) -> io:format("== the demo log tracer ~p started.~nconfig: ~p~n", [InstId, Config]), diff --git a/apps/emqx_resource/examples/log_tracer_schema.erl b/apps/emqx_resource/examples/log_tracer_schema.erl index 2b49aab7f..a8fc55411 100644 --- a/apps/emqx_resource/examples/log_tracer_schema.erl +++ b/apps/emqx_resource/examples/log_tracer_schema.erl @@ -2,7 +2,7 @@ -include_lib("typerefl/include/types.hrl"). --export([fields/1]). +-export([schema/0]). -reflect_type([t_level/0, t_cache_logs_in/0]). @@ -10,15 +10,14 @@ -type t_cache_logs_in() :: memory | file. -fields("config") -> +schema() -> [ {condition, fun condition/1} , {level, fun level/1} , {enable_cache, fun enable_cache/1} , {cache_logs_in, fun cache_logs_in/1} , {cache_log_dir, fun cache_log_dir/1} , {bulk, fun bulk/1} - ]; -fields(_) -> []. + ]. condition(mapping) -> "config.condition"; condition(type) -> map(); diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index c432e683e..b909b0e50 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -37,7 +37,8 @@ %% APIs for instances --export([ parse_config/2 +-export([ check_config/2 + , check_and_load_instance/3 , resource_type_from_str/1 ]). @@ -73,9 +74,6 @@ , list_instances_verbose/0 %% list all the instances , get_instance/1 %% return the data of the instance , get_instance_by_type/1 %% return all the instances of the same resource type - , load_instances_from_dir/1 %% load instances from a directory - , load_instance_from_file/1 %% load an instance from a config file - , load_instance_from_config/1 %% load an instance from a map or json-string config % , dependents/1 % , inc_counter/2 %% increment the counter of the instance % , inc_counter/3 %% increment the counter by a given integer @@ -215,18 +213,6 @@ list_instances_verbose() -> get_instance_by_type(ResourceType) -> emqx_resource_instance:lookup_by_type(ResourceType). --spec load_instances_from_dir(Dir :: string()) -> ok. -load_instances_from_dir(Dir) -> - emqx_resource_instance:load_dir(Dir). - --spec load_instance_from_file(File :: string()) -> ok. -load_instance_from_file(File) -> - emqx_resource_instance:load_file(File). - --spec load_instance_from_config(binary() | map()) -> ok. -load_instance_from_config(Config) -> - emqx_resource_instance:load_config(Config). - -spec call_start(instance_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. call_start(InstId, Mod, Config) -> @@ -267,24 +253,30 @@ call_api_reply_format(Mod, Data) -> true -> ?SAFE_CALL(Mod:on_api_reply_format(Data)) end. --spec parse_config(resource_type(), binary() | term()) -> +-spec check_config(resource_type(), binary() | term()) -> {ok, resource_config()} | {error, term()}. -parse_config(ResourceType, RawConfig) when is_binary(RawConfig) -> +check_config(ResourceType, RawConfig) when is_binary(RawConfig) -> case hocon:binary(RawConfig, #{format => richmap}) of {ok, MapConfig} -> - do_parse_config(ResourceType, MapConfig); + do_check_config(ResourceType, MapConfig); Error -> Error end; -parse_config(ResourceType, RawConfigTerm) -> - parse_config(ResourceType, jsx:encode(#{config => RawConfigTerm})). +check_config(ResourceType, RawConfigTerm) -> + check_config(ResourceType, jsx:encode(#{config => RawConfigTerm})). --spec do_parse_config(resource_type(), map()) -> {ok, resource_config()} | {error, term()}. -do_parse_config(ResourceType, MapConfig) -> - case ?SAFE_CALL(hocon_schema:generate(ResourceType, MapConfig)) of +-spec do_check_config(resource_type(), map()) -> {ok, resource_config()} | {error, term()}. +do_check_config(ResourceType, MapConfig) -> + case ?SAFE_CALL(hocon_schema:check(ResourceType, MapConfig)) of {error, Reason} -> {error, Reason}; - Config -> - InstConf = maps:from_list(proplists:get_value(config, Config)), - {ok, InstConf} + Config -> {ok, maps:get(<<"config">>, hocon:richmap_to_map(Config))} + end. + +-spec check_and_load_instance(instance_id(), resource_type(), binary() | term()) -> + {ok, resource_data()} | {error, term()}. +check_and_load_instance(InstId, ResourceType, Config) -> + case check_config(ResourceType, Config) of + {ok, InstConf} -> emqx_resource_instance:create_local(InstId, ResourceType, InstConf); + Error -> Error end. %% ================================================================================= diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl index 1e19cdede..a92d41b75 100644 --- a/apps/emqx_resource/src/emqx_resource_api.erl +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -46,7 +46,7 @@ put(Mod, #{id := Id}, Params) -> end. do_put(Mod, Id, ConfigParams, ResourceType, Params) -> - case emqx_resource:parse_config(ResourceType, ConfigParams) of + case emqx_resource:check_config(ResourceType, ConfigParams) of {ok, Config} -> case emqx_resource:update(Id, ResourceType, Config, Params) of {ok, Data} -> diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index dbbf052a1..470263a6d 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -23,10 +23,7 @@ -export([start_link/2]). %% load resource instances from *.conf files --export([ load_dir/1 - , load_file/1 - , load_config/1 - , lookup/1 +-export([ lookup/1 , list_all/0 , lookup_by_type/1 , create_local/3 @@ -85,44 +82,8 @@ lookup_by_type(ResourceType) -> [Data || #{mod := Mod} = Data <- list_all() , Mod =:= ResourceType]. --spec load_dir(Dir :: string()) -> ok. -load_dir(Dir) -> - lists:foreach(fun load_file/1, filelib:wildcard(filename:join([Dir, "*.conf"]))). - -load_file(File) -> - case ?SAFE_CALL(hocon_token:read(File)) of - {error, Reason} -> - logger:error("load resource from ~p failed: ~p", [File, Reason]); - RawConfig -> - case load_config(RawConfig) of - {ok, Data} -> - logger:debug("loaded resource instance from file: ~p, data: ~p", - [File, Data]); - {error, Reason} -> - logger:error("load resource from ~p failed: ~p", [File, Reason]) - end - end. - --spec load_config(binary() | map()) -> {ok, resource_data()} | {error, term()}. -load_config(RawConfig) when is_binary(RawConfig) -> - case hocon:binary(RawConfig, #{format => map}) of - {ok, ConfigTerm} -> load_config(ConfigTerm); - Error -> Error - end; - -load_config(#{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr} = Config) -> - MapConfig = maps:get(<<"config">>, Config, #{}), - case emqx_resource:resource_type_from_str(ResourceTypeStr) of - {ok, ResourceType} -> parse_and_load_config(Id, ResourceType, MapConfig); - Error -> Error - end. - -parse_and_load_config(InstId, ResourceType, MapConfig) -> - case emqx_resource:parse_config(ResourceType, MapConfig) of - {ok, InstConf} -> create_local(InstId, ResourceType, InstConf); - Error -> Error - end. - +-spec create_local(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, term()}. create_local(InstId, ResourceType, InstConf) -> case hash_call(InstId, {create, InstId, ResourceType, InstConf}, 15000) of {ok, Data} -> {ok, Data}; diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl index cd6c7e4ae..6207d675e 100644 --- a/apps/emqx_resource/src/emqx_resource_transform.erl +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -68,12 +68,18 @@ form(Mod, Form) -> fix_spec_attrs() -> [ ?Q("-export([emqx_resource_schema/0]).") - , ?Q("-export([structs/0]).") + , ?Q("-export([structs/0, fields/1]).") , ?Q("-behaviour(hocon_schema).") ]. fix_spec_funcs(_Mod) -> - [ (?Q("emqx_resource_schema() -> <<\"demo_swagger_schema\">>.")) + [ ?Q("emqx_resource_schema() -> <<\"demo_swagger_schema\">>.") , ?Q("structs() -> [\"config\"].") + , ?Q("fields(\"config\") -> " + "[fun (type) -> \"schema\"; " + " (_) -> undefined " + " end];" + "fields(\"schema\") -> schema()." + ) ]. fix_api_attrs(Mod, Path) -> diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index eb4e58d1f..5ac46e0e3 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -7,4 +7,5 @@ {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. {emqx_resource, {{enable_plugin_emqx_resource}}}. {emqx_connector, {{enable_plugin_emqx_connector}}}. +{emqx_data_bridge, {{enable_plugin_emqx_data_bridge}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index fa5586ad5..c5206e9e2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -182,6 +182,7 @@ overlay_vars_rel(RelType) -> , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} , {enable_plugin_emqx_resource, true} , {enable_plugin_emqx_connector, true} + , {enable_plugin_emqx_data_bridge, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -277,6 +278,7 @@ relx_plugin_apps(ReleaseType) -> , emqx_recon , emqx_resource , emqx_connector + , emqx_data_bridge , emqx_rule_engine , emqx_sasl ] From 4f451ee8627d0b004c7f960236ec6b9236b6ede6 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 5 Jun 2021 15:30:21 +0800 Subject: [PATCH 10/19] feat(emqx_resource): delete API generation from parse transfrom --- .../src/emqx_resource_transform.erl | 54 +++---------------- 1 file changed, 7 insertions(+), 47 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl index 6207d675e..21327dd9f 100644 --- a/apps/emqx_resource/src/emqx_resource_transform.erl +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -47,23 +47,20 @@ trans(Mod, Forms) -> forms(Mod, [F0 | Fs0]) -> case form(Mod, F0) of - {CurrForm, AppendedForms} -> - CurrForm ++ forms(Mod, Fs0) ++ AppendedForms; - {AHeadForms, CurrForm, AppendedForms} -> - AHeadForms ++ CurrForm ++ forms(Mod, Fs0) ++ AppendedForms + {CurrForms, AppendedForms} -> + CurrForms ++ forms(Mod, Fs0) ++ AppendedForms; + {CurrForms, FollowerForms, AppendedForms} -> + CurrForms ++ FollowerForms ++ forms(Mod, Fs0) ++ AppendedForms end; forms(_, []) -> []. form(Mod, Form) -> case Form of - ?Q("-emqx_resource_api_path('@Path').") -> - {fix_spec_attrs() ++ fix_api_attrs(Mod, erl_syntax:concrete(Path)) - ++ fix_api_exports(), - [], - fix_spec_funcs(Mod) ++ fix_api_funcs(Mod)}; + ?Q("-module('@_').") -> + {[Form], fix_spec_attrs(), fix_spec_funcs(Mod)}; _ -> %io:format("---other form: ~p~n", [Form]), - {[], [Form], []} + {[Form], [], []} end. fix_spec_attrs() -> @@ -81,40 +78,3 @@ fix_spec_funcs(_Mod) -> "fields(\"schema\") -> schema()." ) ]. - -fix_api_attrs(Mod, Path) -> - BaseName = atom_to_list(Mod), - [erl_syntax:revert( - erl_syntax:attribute(?Q("rest_api"), [ - erl_syntax:abstract(#{ - name => list_to_atom(Act ++ "_" ++ BaseName), - method => Method, - path => mk_path(Path, WithId), - func => Func, - descr => Act ++ " the " ++ BaseName})])) - || {Act, Method, WithId, Func} <- [ - {"list", 'GET', noid, api_get_all}, - {"get", 'GET', id, api_get}, - {"update", 'PUT', id, api_put}, - {"delete", 'DELETE', id, api_delete}]]. - -fix_api_exports() -> - [?Q("-export([api_get_all/2, api_get/2, api_put/2, api_delete/2]).")]. - -fix_api_funcs(Mod) -> - [erl_syntax:revert(?Q( - "api_get_all(Binding, Params) -> - emqx_resource_api:get_all('@Mod@', Binding, Params).")), - erl_syntax:revert(?Q( - "api_get(Binding, Params) -> - emqx_resource_api:get('@Mod@', Binding, Params).")), - erl_syntax:revert(?Q( - "api_put(Binding, Params) -> - emqx_resource_api:put('@Mod@', Binding, Params).")), - erl_syntax:revert(?Q( - "api_delete(Binding, Params) -> - emqx_resource_api:delete('@Mod@', Binding, Params).")) - ]. - -mk_path(Path, id) -> string:trim(Path, trailing, "/") ++ "/:bin:id"; -mk_path(Path, noid) -> Path. From 2ff92d28807de0ebe42b48eace55789c86b9cdf7 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 16:12:06 +0800 Subject: [PATCH 11/19] feat(emqx_data_bridge): add HTTP APIs for data bridge --- .../src/emqx_connector_mysql.erl | 2 - .../emqx_data_bridge/src/emqx_data_bridge.erl | 7 ++ .../src/emqx_data_bridge_api.erl | 98 +++++++++++++++++++ .../src/emqx_data_bridge_monitor.erl | 6 +- apps/emqx_resource/src/emqx_resource.erl | 72 +++++++++++--- apps/emqx_resource/src/emqx_resource_api.erl | 59 ++--------- .../src/emqx_resource_instance.erl | 2 +- 7 files changed, 176 insertions(+), 70 deletions(-) create mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge_api.erl diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 1bc1bbc80..a00de8426 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -18,8 +18,6 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). --emqx_resource_api_path("connectors/mysql"). - -export([ on_jsonify/1 , on_api_reply_format/1 ]). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index cd26b9d3e..0f6c42246 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -1,9 +1,16 @@ -module(emqx_data_bridge). -export([ load_bridges/0 + , resource_type/1 + , resource_id/1 ]). load_bridges() -> Bridges = proplists:get_value(bridges, application:get_all_env(emqx_data_bridge), []), emqx_data_bridge_monitor:ensure_all_started(Bridges). + +resource_type(<<"mysql">>) -> emqx_connector_mysql. + +resource_id(BridgeName) -> + <<"bridge:", BridgeName/binary>>. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl new file mode 100644 index 000000000..02b1edb99 --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl @@ -0,0 +1,98 @@ +-module(emqx_data_bridge_api). + +-rest_api(#{ name => list_data_bridges + , method => 'GET' + , path => "/data_bridges" + , func => list_bridges + , descr => "List all data bridges" + }). + +-rest_api(#{ name => get_data_bridge + , method => 'GET' + , path => "/data_bridges/:bin:name" + , func => get_bridge + , descr => "Get a data bridge by name" + }). + +-rest_api(#{ name => create_data_bridge + , method => 'POST' + , path => "/data_bridges/:bin:name" + , func => create_bridge + , descr => "Create a new data bridge" + }). + +-rest_api(#{ name => update_data_bridge + , method => 'POST' + , path => "/data_bridges/:bin:name" + , func => update_bridge + , descr => "Update an existing data bridge" + }). + +-rest_api(#{ name => delete_data_bridge + , method => 'DELETE' + , path => "/data_bridges/:bin:name" + , func => delete_bridge + , descr => "Delete an existing data bridge" + }). + +-export([ list_bridges/2 + , get_bridge/2 + , create_bridge/2 + , update_bridge/2 + , delete_bridge/2 + ]). + +list_bridges(_Binding, _Params) -> + {200, #{code => 0, data => emqx_resource_api:list_instances(fun is_bridge/1)}}. + +get_bridge(#{name := Name}, _Params) -> + case emqx_resource:get_instance(emqx_data_bridge:resource_id(Name)) of + {ok, Data} -> + {200, #{code => 0, data => emqx_resource_api:format_data(Data)}}; + {error, not_found} -> + {404, #{code => 102, message => <<"not_found: ", Name/binary>>}} + end. + +create_bridge(#{name := Name}, Params) -> + Config = proplists:get_value(<<"config">>, Params), + BridgeType = proplists:get_value(<<"type">>, Params), + case emqx_resource:check_and_create( + emqx_data_bridge:resource_id(Name), + emqx_data_bridge:resource_type(BridgeType), Config) of + {ok, Data} -> + {200, #{code => 0, data => emqx_resource_api:format_data(Data)}}; + {error, already_created} -> + {400, #{code => 102, message => <<"bridge already created: ", Name/binary>>}}; + {error, Reason0} -> + Reason = emqx_data_bridge_api:stringnify(Reason0), + {500, #{code => 102, message => <<"create bridge ", Name/binary, + " failed:", Reason/binary>>}} + end. + +update_bridge(#{name := Name}, Params) -> + Config = proplists:get_value(<<"config">>, Params), + BridgeType = proplists:get_value(<<"type">>, Params), + case emqx_resource:check_and_udpate( + emqx_data_bridge:resource_id(Name), + emqx_data_bridge:resource_type(BridgeType), Config, []) of + {ok, Data} -> + {200, #{code => 0, data => emqx_resource_api:format_data(Data)}}; + {error, not_found} -> + {400, #{code => 102, message => <<"bridge not_found: ", Name/binary>>}}; + {error, Reason0} -> + Reason = emqx_data_bridge_api:stringnify(Reason0), + {500, #{code => 102, message => <<"update bridge ", Name/binary, + " failed:", Reason/binary>>}} + end. + +delete_bridge(#{name := Name}, _Params) -> + case emqx_resource:remove(emqx_data_bridge:resource_id(Name)) of + ok -> {200, #{code => 0, data => #{}}}; + {error, Reason} -> + {500, #{code => 102, message => emqx_data_bridge_api:stringnify(Reason)}} + end. + +is_bridge(#{id := <<"bridge:", _/binary>>}) -> + true; +is_bridge(_Data) -> + false. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl index 22c216038..6073602b9 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl @@ -54,11 +54,11 @@ load_bridges(Configs) -> load_bridge(#{<<"name">> := Name, <<"type">> := Type, <<"config">> := Config}) -> - case emqx_resource:check_and_load_instance(Name, resource_type(Type), Config) of + case emqx_resource:check_and_create_local( + emqx_data_bridge:resource_id(Name), + emqx_data_bridge:resource_type(Type), Config) of {ok, _} -> ok; {error, already_created} -> ok; {error, Reason} -> error({load_bridge, Reason}) end. - -resource_type(<<"mysql">>) -> emqx_connector_mysql. \ No newline at end of file diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b909b0e50..8a974fe36 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -38,7 +38,10 @@ %% APIs for instances -export([ check_config/2 - , check_and_load_instance/3 + , check_and_create/3 + , check_and_create_local/3 + , check_and_update/4 + , check_and_update_local/4 , resource_type_from_str/1 ]). @@ -46,10 +49,13 @@ %% provisional solution: rpc:multical to all the nodes for creating/updating/removing %% todo: replicate operations -export([ create/3 %% store the config and start the instance + , create_local/3 , create_dry_run/3 %% run start/2, health_check/2 and stop/1 sequentially + , create_dry_run_local/3 , update/4 %% update the config, stop the old instance and start the new one - %% it will create a new resource when the id does not exist + , update_local/4 , remove/1 %% remove the config and stop the instance + , remove_local/1 ]). %% Calls to the callback module with current resource state @@ -73,7 +79,7 @@ -export([ list_instances/0 %% list all the instances, id only. , list_instances_verbose/0 %% list all the instances , get_instance/1 %% return the data of the instance - , get_instance_by_type/1 %% return all the instances of the same resource type + , list_instances_by_type/1 %% return all the instances of the same resource type % , dependents/1 % , inc_counter/2 %% increment the counter of the instance % , inc_counter/3 %% increment the counter by a given integer @@ -152,22 +158,42 @@ query_failed({_, {OnFailed, Args}}) -> -spec create(instance_id(), resource_type(), resource_config()) -> {ok, resource_data()} | {error, Reason :: term()}. create(InstId, ResourceType, Config) -> - ?CLUSTER_CALL(call_instance, [InstId, {create, InstId, ResourceType, Config}], {ok, _}). + ?CLUSTER_CALL(create_local, [InstId, ResourceType, Config], {ok, _}). + +-spec create_local(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, Reason :: term()}. +create_local(InstId, ResourceType, Config) -> + call_instance(InstId, {create, InstId, ResourceType, Config}). -spec create_dry_run(instance_id(), resource_type(), resource_config()) -> ok | {error, Reason :: term()}. create_dry_run(InstId, ResourceType, Config) -> - ?CLUSTER_CALL(call_instance, [InstId, {create_dry_run, InstId, ResourceType, Config}]). + ?CLUSTER_CALL(create_dry_run_local, [InstId, ResourceType, Config]). + +-spec create_dry_run_local(instance_id(), resource_type(), resource_config()) -> + ok | {error, Reason :: term()}. +create_dry_run_local(InstId, ResourceType, Config) -> + call_instance(InstId, {create_dry_run, InstId, ResourceType, Config}). -spec update(instance_id(), resource_type(), resource_config(), term()) -> {ok, resource_data()} | {error, Reason :: term()}. update(InstId, ResourceType, Config, Params) -> - ?CLUSTER_CALL(call_instance, [InstId, {update, InstId, ResourceType, Config, Params}], {ok, _}). + ?CLUSTER_CALL(update_local, [InstId, ResourceType, Config, Params], {ok, _}). + +-spec update_local(instance_id(), resource_type(), resource_config(), term()) -> + {ok, resource_data()} | {error, Reason :: term()}. +update_local(InstId, ResourceType, Config, Params) -> + call_instance(InstId, {update, InstId, ResourceType, Config, Params}). -spec remove(instance_id()) -> ok | {error, Reason :: term()}. remove(InstId) -> - ?CLUSTER_CALL(call_instance, [InstId, {remove, InstId}]). + ?CLUSTER_CALL(remove_local, [InstId]). +-spec remove_local(instance_id()) -> ok | {error, Reason :: term()}. +remove_local(InstId) -> + call_instance(InstId, {remove, InstId}). + +%% ================================================================================= -spec query(instance_id(), Request :: term()) -> Result :: term(). query(InstId, Request) -> query(InstId, Request, undefined). @@ -209,8 +235,8 @@ list_instances() -> list_instances_verbose() -> emqx_resource_instance:list_all(). --spec get_instance_by_type(module()) -> [resource_data()]. -get_instance_by_type(ResourceType) -> +-spec list_instances_by_type(module()) -> [resource_data()]. +list_instances_by_type(ResourceType) -> emqx_resource_instance:lookup_by_type(ResourceType). -spec call_start(instance_id(), module(), resource_config()) -> @@ -271,11 +297,33 @@ do_check_config(ResourceType, MapConfig) -> Config -> {ok, maps:get(<<"config">>, hocon:richmap_to_map(Config))} end. --spec check_and_load_instance(instance_id(), resource_type(), binary() | term()) -> +-spec check_and_create(instance_id(), resource_type(), binary() | term()) -> {ok, resource_data()} | {error, term()}. -check_and_load_instance(InstId, ResourceType, Config) -> +check_and_create(InstId, ResourceType, Config) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> create(InstId, ResourceType, InstConf) end). + +-spec check_and_create_local(instance_id(), resource_type(), binary() | term()) -> + {ok, resource_data()} | {error, term()}. +check_and_create_local(InstId, ResourceType, Config) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> create_local(InstId, ResourceType, InstConf) end). + +-spec check_and_update(instance_id(), resource_type(), binary() | term(), term()) -> + {ok, resource_data()} | {error, term()}. +check_and_update(InstId, ResourceType, Config, Params) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> update(InstId, ResourceType, InstConf, Params) end). + +-spec check_and_update_local(instance_id(), resource_type(), binary() | term(), term()) -> + {ok, resource_data()} | {error, term()}. +check_and_update_local(InstId, ResourceType, Config, Params) -> + check_and_do(ResourceType, Config, + fun(InstConf) -> update_local(InstId, ResourceType, InstConf, Params) end). + +check_and_do(ResourceType, Config, Do) when is_function(Do) -> case check_config(ResourceType, Config) of - {ok, InstConf} -> emqx_resource_instance:create_local(InstId, ResourceType, InstConf); + {ok, InstConf} -> Do(InstConf); Error -> Error end. diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl index a92d41b75..fe1ca4509 100644 --- a/apps/emqx_resource/src/emqx_resource_api.erl +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -15,61 +15,16 @@ %%-------------------------------------------------------------------- -module(emqx_resource_api). --export([ get_all/3 - , get/3 - , put/3 - , delete/3 +-export([ list_instances/1 + , format_data/1 + , stringnify/1 ]). --export([default_api_reply_format/1]). +list_instances(Filter) -> + [format_data(Data) || Data <- emqx_resource:list_instances_verbose(), Filter(Data)]. -get_all(Mod, _Binding, _Params) -> - {200, #{code => 0, data => - [format_data(Mod, Data) || Data <- emqx_resource:list_instances_verbose()]}}. - -get(Mod, #{id := Id}, _Params) -> - case emqx_resource:get_instance(stringnify(Id)) of - {ok, Data} -> - {200, #{code => 0, data => format_data(Mod, Data)}}; - {error, not_found} -> - {404, #{code => 102, message => {resource_instance_not_found, stringnify(Id)}}} - end. - -put(Mod, #{id := Id}, Params) -> - ConfigParams = proplists:get_value(<<"config">>, Params), - ResourceTypeStr = proplists:get_value(<<"resource_type">>, Params, #{}), - case emqx_resource:resource_type_from_str(ResourceTypeStr) of - {ok, ResourceType} -> - do_put(Mod, stringnify(Id), ConfigParams, ResourceType, Params); - {error, Reason} -> - {404, #{code => 102, message => stringnify(Reason)}} - end. - -do_put(Mod, Id, ConfigParams, ResourceType, Params) -> - case emqx_resource:check_config(ResourceType, ConfigParams) of - {ok, Config} -> - case emqx_resource:update(Id, ResourceType, Config, Params) of - {ok, Data} -> - {200, #{code => 0, data => format_data(Mod, Data)}}; - {error, Reason} -> - {500, #{code => 102, message => stringnify(Reason)}} - end; - {error, Reason} -> - {400, #{code => 108, message => stringnify(Reason)}} - end. - -delete(_Mod, #{id := Id}, _Params) -> - case emqx_resource:remove(stringnify(Id)) of - ok -> {200, #{code => 0, data => #{}}}; - {error, Reason} -> - {500, #{code => 102, message => stringnify(Reason)}} - end. - -format_data(Mod, Data) -> - emqx_resource:call_api_reply_format(Mod, Data). - -default_api_reply_format(#{id := Id, mod := Mod, status := Status, config := Config}) -> - #{node => node(), id => Id, status => Status, resource_type => Mod, +format_data(#{id := Id, mod := Mod, status := Status, config := Config}) -> + #{id => Id, status => Status, resource_type => Mod, config => emqx_resource:call_jsonify(Mod, Config)}. stringnify(Bin) when is_binary(Bin) -> Bin; diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 470263a6d..1e924e249 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -167,7 +167,7 @@ do_update(InstId, ResourceType, NewConfig, Params) -> {ok, #{mod := Mod}} when Mod =/= ResourceType -> {error, updating_to_incorrect_resource_type}; {error, not_found} -> - do_create(InstId, ResourceType, NewConfig) + {error, not_found} end. do_create(InstId, ResourceType, Config) -> From 4914b003ac74901659b1877d6b9a37803d394282 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 17:20:34 +0800 Subject: [PATCH 12/19] feat(emqx_resource): update the unused APIs --- apps/emqx_connector/src/emqx_connector_mysql.erl | 6 ------ apps/emqx_data_bridge/src/emqx_data_bridge_api.erl | 8 ++++---- apps/emqx_resource/src/emqx_resource.erl | 8 -------- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index a00de8426..423a6edf1 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -19,7 +19,6 @@ -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). -export([ on_jsonify/1 - , on_api_reply_format/1 ]). %% callbacks of behaviour emqx_resource @@ -51,11 +50,6 @@ on_jsonify(#{<<"server">> := Server, <<"user">> := User, <<"database">> := DB, <<"certfile">> => list_to_binary(CertFile) }. -on_api_reply_format(ResourceData) -> - #{config := Conf} = Reply0 = emqx_resource_api:default_api_reply_format(ResourceData), - Reply0#{config => maps:without([<<"cacertfile">>, <<"keyfile">>, - <<"certfile">>, <<"verify">>], Conf)}. - %% =================================================================== on_start(InstId, #{<<"server">> := {Host, Port}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl index 02b1edb99..ceb4201e1 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl @@ -64,7 +64,7 @@ create_bridge(#{name := Name}, Params) -> {error, already_created} -> {400, #{code => 102, message => <<"bridge already created: ", Name/binary>>}}; {error, Reason0} -> - Reason = emqx_data_bridge_api:stringnify(Reason0), + Reason = emqx_resource_api:stringnify(Reason0), {500, #{code => 102, message => <<"create bridge ", Name/binary, " failed:", Reason/binary>>}} end. @@ -72,7 +72,7 @@ create_bridge(#{name := Name}, Params) -> update_bridge(#{name := Name}, Params) -> Config = proplists:get_value(<<"config">>, Params), BridgeType = proplists:get_value(<<"type">>, Params), - case emqx_resource:check_and_udpate( + case emqx_resource:check_and_update( emqx_data_bridge:resource_id(Name), emqx_data_bridge:resource_type(BridgeType), Config, []) of {ok, Data} -> @@ -80,7 +80,7 @@ update_bridge(#{name := Name}, Params) -> {error, not_found} -> {400, #{code => 102, message => <<"bridge not_found: ", Name/binary>>}}; {error, Reason0} -> - Reason = emqx_data_bridge_api:stringnify(Reason0), + Reason = emqx_resource_api:stringnify(Reason0), {500, #{code => 102, message => <<"update bridge ", Name/binary, " failed:", Reason/binary>>}} end. @@ -89,7 +89,7 @@ delete_bridge(#{name := Name}, _Params) -> case emqx_resource:remove(emqx_data_bridge:resource_id(Name)) of ok -> {200, #{code => 0, data => #{}}}; {error, Reason} -> - {500, #{code => 102, message => emqx_data_bridge_api:stringnify(Reason)}} + {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} end. is_bridge(#{id := <<"bridge:", _/binary>>}) -> diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 8a974fe36..97d71fd4b 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -73,7 +73,6 @@ , call_stop/3 %% stop the instance , call_config_merge/4 %% merge the config when updating , call_jsonify/2 - , call_api_reply_format/2 ]). -export([ list_instances/0 %% list all the instances, id only. @@ -272,13 +271,6 @@ call_jsonify(Mod, Config) -> true -> ?SAFE_CALL(Mod:on_jsonify(Config)) end. --spec call_api_reply_format(module(), resource_data()) -> jsx:json_term(). -call_api_reply_format(Mod, Data) -> - case erlang:function_exported(Mod, on_api_reply_format, 1) of - false -> emqx_resource_api:default_api_reply_format(Data); - true -> ?SAFE_CALL(Mod:on_api_reply_format(Data)) - end. - -spec check_config(resource_type(), binary() | term()) -> {ok, resource_config()} | {error, term()}. check_config(ResourceType, RawConfig) when is_binary(RawConfig) -> From e9fe345ac8f44850e44bd4b787a70281964426c3 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 19:52:08 +0800 Subject: [PATCH 13/19] fix(emqx_data_bridge): correct the format of HTTP responses --- .../emqx_data_bridge/src/emqx_data_bridge.erl | 18 ++++++- .../src/emqx_data_bridge_api.erl | 24 ++++----- .../src/emqx_data_bridge_config_handler.erl | 54 +++++++++++++++++++ .../src/emqx_data_bridge_monitor.erl | 2 +- 4 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index 0f6c42246..a5a3087dc 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -2,7 +2,10 @@ -export([ load_bridges/0 , resource_type/1 - , resource_id/1 + , name_to_resource_id/1 + , resource_id_to_name/1 + , list_bridges/0 + , is_bridge/1 ]). load_bridges() -> @@ -12,5 +15,16 @@ load_bridges() -> resource_type(<<"mysql">>) -> emqx_connector_mysql. -resource_id(BridgeName) -> +name_to_resource_id(BridgeName) -> <<"bridge:", BridgeName/binary>>. + +resource_id_to_name(<<"bridge:", BridgeName/binary>> = _ResourceId) -> + BridgeName. + +list_bridges() -> + emqx_resource_api:list_instances(fun emqx_data_bridge:is_bridge/1). + +is_bridge(#{id := <<"bridge:", _/binary>>}) -> + true; +is_bridge(_Data) -> + false. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl index ceb4201e1..7d2f5274e 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl @@ -43,12 +43,13 @@ ]). list_bridges(_Binding, _Params) -> - {200, #{code => 0, data => emqx_resource_api:list_instances(fun is_bridge/1)}}. + {200, #{code => 0, data => [format_api_reply(Data) || + Data <- emqx_data_bridge:list_bridges()]}}. get_bridge(#{name := Name}, _Params) -> - case emqx_resource:get_instance(emqx_data_bridge:resource_id(Name)) of + case emqx_resource:get_instance(emqx_data_bridge:name_to_resource_id(Name)) of {ok, Data} -> - {200, #{code => 0, data => emqx_resource_api:format_data(Data)}}; + {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; {error, not_found} -> {404, #{code => 102, message => <<"not_found: ", Name/binary>>}} end. @@ -57,10 +58,10 @@ create_bridge(#{name := Name}, Params) -> Config = proplists:get_value(<<"config">>, Params), BridgeType = proplists:get_value(<<"type">>, Params), case emqx_resource:check_and_create( - emqx_data_bridge:resource_id(Name), + emqx_data_bridge:name_to_resource_id(Name), emqx_data_bridge:resource_type(BridgeType), Config) of {ok, Data} -> - {200, #{code => 0, data => emqx_resource_api:format_data(Data)}}; + {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; {error, already_created} -> {400, #{code => 102, message => <<"bridge already created: ", Name/binary>>}}; {error, Reason0} -> @@ -73,10 +74,10 @@ update_bridge(#{name := Name}, Params) -> Config = proplists:get_value(<<"config">>, Params), BridgeType = proplists:get_value(<<"type">>, Params), case emqx_resource:check_and_update( - emqx_data_bridge:resource_id(Name), + emqx_data_bridge:name_to_resource_id(Name), emqx_data_bridge:resource_type(BridgeType), Config, []) of {ok, Data} -> - {200, #{code => 0, data => emqx_resource_api:format_data(Data)}}; + {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; {error, not_found} -> {400, #{code => 102, message => <<"bridge not_found: ", Name/binary>>}}; {error, Reason0} -> @@ -86,13 +87,12 @@ update_bridge(#{name := Name}, Params) -> end. delete_bridge(#{name := Name}, _Params) -> - case emqx_resource:remove(emqx_data_bridge:resource_id(Name)) of + case emqx_resource:remove(emqx_data_bridge:name_to_resource_id(Name)) of ok -> {200, #{code => 0, data => #{}}}; {error, Reason} -> {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} end. -is_bridge(#{id := <<"bridge:", _/binary>>}) -> - true; -is_bridge(_Data) -> - false. +format_api_reply(#{resource_type := Type, id := Id, config := Conf, status := Status}) -> + #{type => Type, name => emqx_data_bridge:resource_id_to_name(Id), + config => Conf, status => Status}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl new file mode 100644 index 000000000..f4efb0754 --- /dev/null +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl @@ -0,0 +1,54 @@ +-module(emqx_data_bridge_config_handler). + +-behaviour(gen_server). + +%% API functions +-export([ start_link/0 + , notify_updated/0 + ]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-record(state, {}). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +notify_updated() -> + gen_server:cast(?MODULE, updated). + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(updated, State) -> + Configs = [format_conf(Data) || Data <- emqx_data_bridge:list_bridges()], + emqx_config_handler ! {emqx_data_bridge, Configs}, + {noreply, State}; + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%============================================================================ + +format_conf(#{resource_type := Type, id := Id, config := Conf}) -> + #{type => Type, name => emqx_data_bridge:resource_id_to_name(Id), + config => Conf}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl index 6073602b9..243d81237 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl @@ -55,7 +55,7 @@ load_bridges(Configs) -> load_bridge(#{<<"name">> := Name, <<"type">> := Type, <<"config">> := Config}) -> case emqx_resource:check_and_create_local( - emqx_data_bridge:resource_id(Name), + emqx_data_bridge:name_to_resource_id(Name), emqx_data_bridge:resource_type(Type), Config) of {ok, _} -> ok; {error, already_created} -> ok; From dddab3132698ab019047c5b49190978923747224 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 20:01:28 +0800 Subject: [PATCH 14/19] fix(emqx_data_bridge): resource type to bridge type --- apps/emqx_data_bridge/src/emqx_data_bridge.erl | 3 +++ apps/emqx_data_bridge/src/emqx_data_bridge_api.erl | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index a5a3087dc..e2707924b 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -2,6 +2,7 @@ -export([ load_bridges/0 , resource_type/1 + , bridge_type/1 , name_to_resource_id/1 , resource_id_to_name/1 , list_bridges/0 @@ -15,6 +16,8 @@ load_bridges() -> resource_type(<<"mysql">>) -> emqx_connector_mysql. +bridge_type(emqx_connector_mysql) -> <<"mysql">>. + name_to_resource_id(BridgeName) -> <<"bridge:", BridgeName/binary>>. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl index 7d2f5274e..b0124e149 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl @@ -94,5 +94,6 @@ delete_bridge(#{name := Name}, _Params) -> end. format_api_reply(#{resource_type := Type, id := Id, config := Conf, status := Status}) -> - #{type => Type, name => emqx_data_bridge:resource_id_to_name(Id), + #{type => emqx_data_bridge:bridge_type(Type), + name => emqx_data_bridge:resource_id_to_name(Id), config => Conf, status => Status}. From 16c0a5b80a22bf4b9037b8726317847276e1f78d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 21:25:19 +0800 Subject: [PATCH 15/19] fix(emqx_resource): merge conflict --- apps/emqx_resource/src/emqx_resource_instance.erl | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index d2f4dbbfa..1e924e249 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -100,16 +100,6 @@ save_config_to_disk(InstId, ResourceType, Config) -> emqx_data_dir() -> "data". -save_config_to_disk(InstId, ResourceType, Config) -> - %% TODO: send an event to the config handler, and the hander (single process) - %% will dump configs for all instances (from an ETS table) to a file. - file:write_file(filename:join([emqx_data_dir(), binary_to_list(InstId) ++ ".conf"]), - jsx:encode(#{id => InstId, resource_type => ResourceType, - config => emqx_resource:call_jsonify(ResourceType, Config)})). - -emqx_data_dir() -> - "data". - %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ From 3c1c45769763c9941732b6b6e7a6789ca3ca3a33 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 21:54:17 +0800 Subject: [PATCH 16/19] fix(emqx_resource): call to undefined function hocon:richmap_to_map/1 --- apps/emqx_resource/src/emqx_resource.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 97d71fd4b..b2bbc586f 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -286,7 +286,7 @@ check_config(ResourceType, RawConfigTerm) -> do_check_config(ResourceType, MapConfig) -> case ?SAFE_CALL(hocon_schema:check(ResourceType, MapConfig)) of {error, Reason} -> {error, Reason}; - Config -> {ok, maps:get(<<"config">>, hocon:richmap_to_map(Config))} + Config -> {ok, maps:get(<<"config">>, hocon_schema:richmap_to_map(Config))} end. -spec check_and_create(instance_id(), resource_type(), binary() | term()) -> From 149a54cef457b6cd2981337aab2dbfd93350611c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 22:08:21 +0800 Subject: [PATCH 17/19] fix(emqx_resource): dialyzer issues --- apps/emqx_resource/src/emqx_resource_transform.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl index e18bf1eab..34c6633ef 100644 --- a/apps/emqx_resource/src/emqx_resource_transform.erl +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -22,7 +22,7 @@ parse_transform(Forms, _Opts) -> Mod = hd([M || {attribute, _, module, M} <- Forms]), AST = trans(Mod, proplists:delete(eof, Forms)), - debug_print(Mod, AST), + _ = debug_print(Mod, AST), AST. -ifdef(RESOURCE_DEBUG). From bdf6374414fd3275d1940f37a7c039f3f8e9d23e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 22:11:05 +0800 Subject: [PATCH 18/19] fix(emqx_resource): disable the debug print for parse-transformed code --- apps/emqx_resource/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 66271ee56..4e88043de 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -1,6 +1,6 @@ {erl_opts, [ debug_info , nowarn_unused_import - , {d, 'RESOURCE_DEBUG'} + %, {d, 'RESOURCE_DEBUG'} ]}. {erl_first_files, ["src/emqx_resource_transform.erl"]}. From 2f720f4a17655f654a6c4872ed1aafef6588ef5e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 7 Jun 2021 22:15:23 +0800 Subject: [PATCH 19/19] feat(emqx_data_bridge): add license headers --- apps/emqx_data_bridge/src/emqx_data_bridge.erl | 15 +++++++++++++++ .../emqx_data_bridge/src/emqx_data_bridge_api.erl | 15 +++++++++++++++ .../src/emqx_data_bridge_config_handler.erl | 15 +++++++++++++++ .../src/emqx_data_bridge_monitor.erl | 15 +++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index e2707924b..2adc48111 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_data_bridge). -export([ load_bridges/0 diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl index b0124e149..4de432ca8 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_data_bridge_api). -rest_api(#{ name => list_data_bridges diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl index f4efb0754..387f2b153 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_config_handler.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_data_bridge_config_handler). -behaviour(gen_server). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl index 243d81237..f3b6e6736 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- %% This process monitors all the data bridges, and try to restart a bridge %% when one of it stopped. -module(emqx_data_bridge_monitor).