From 85a130f68b6ea17ec8f39e3f430ae266f8cd603f Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 27 Jun 2023 15:30:23 +0200 Subject: [PATCH] refactor: move MySQL open source connector to its own app --- apps/emqx_authn/rebar.config | 3 ++- apps/emqx_authn/src/emqx_authn.app.src | 3 ++- .../src/simple_authn/emqx_authn_mysql.erl | 6 ++--- .../test/emqx_authn_mysql_SUITE.erl | 2 +- apps/emqx_authz/rebar.config | 3 ++- apps/emqx_authz/src/emqx_authz.app.src | 3 ++- apps/emqx_authz/src/emqx_authz_api_schema.erl | 2 +- apps/emqx_authz/src/emqx_authz_mysql.erl | 4 ++-- apps/emqx_authz/src/emqx_authz_schema.erl | 2 +- .../test/emqx_authz_mysql_SUITE.erl | 2 +- apps/emqx_connector/README.md | 23 +------------------ apps/emqx_connector/docker-ct | 1 - apps/emqx_connector/rebar.config | 1 - .../emqx_connector/src/emqx_connector.app.src | 2 -- apps/emqx_mysql/README.md | 15 ++++++++++++ apps/emqx_mysql/docker-ct | 1 + apps/emqx_mysql/rebar.config | 9 ++++++++ apps/emqx_mysql/src/emqx_mysql.app.src | 14 +++++++++++ .../src/emqx_mysql.erl} | 4 ++-- .../test/emqx_mysql_SUITE.erl} | 8 +++---- lib-ee/emqx_ee_bridge/rebar.config | 1 + lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 2 +- .../src/emqx_ee_bridge_mysql.erl | 2 +- mix.exs | 1 + rebar.config.erl | 1 + ...connector_mysql.hocon => emqx_mysql.hocon} | 2 +- 26 files changed, 69 insertions(+), 48 deletions(-) create mode 100644 apps/emqx_mysql/README.md create mode 100644 apps/emqx_mysql/docker-ct create mode 100644 apps/emqx_mysql/rebar.config create mode 100644 apps/emqx_mysql/src/emqx_mysql.app.src rename apps/{emqx_connector/src/emqx_connector_mysql.erl => emqx_mysql/src/emqx_mysql.erl} (99%) rename apps/{emqx_connector/test/emqx_connector_mysql_SUITE.erl => emqx_mysql/test/emqx_mysql_SUITE.erl} (97%) rename rel/i18n/{emqx_connector_mysql.hocon => emqx_mysql.hocon} (91%) diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 163d491c9..5b0cffb1e 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -5,7 +5,8 @@ {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}}, {emqx_mongodb, {path, "../emqx_mongodb"}}, - {emqx_redis, {path, "../emqx_redis"}} + {emqx_redis, {path, "../emqx_redis"}}, + {emqx_mysql, {path, "../emqx_mysql"}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 31324d52c..30ab93f8c 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -14,7 +14,8 @@ mysql, jose, emqx_mongodb, - emqx_redis + emqx_redis, + emqx_mysql ]}, {mod, {emqx_authn_app, []}}, {env, []}, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index d8e631885..dc4e0d163 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -62,7 +62,7 @@ fields(mysql) -> {query, fun query/1}, {query_timeout, fun query_timeout/1} ] ++ emqx_authn_schema:common_fields() ++ - proplists:delete(prepare_statement, emqx_connector_mysql:fields(config)). + proplists:delete(prepare_statement, emqx_mysql:fields(config)). desc(mysql) -> ?DESC(mysql); @@ -92,12 +92,12 @@ create(_AuthenticatorID, Config) -> create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), - {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_connector_mysql, Config), + {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config), {ok, State#{resource_id => ResourceId}}. update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_connector_mysql, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 9d2c5cf63..914ce4dd1 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -62,7 +62,7 @@ init_per_suite(Config) -> {ok, _} = emqx_resource:create_local( ?MYSQL_RESOURCE, ?RESOURCE_GROUP, - emqx_connector_mysql, + emqx_mysql, mysql_config(), #{} ), diff --git a/apps/emqx_authz/rebar.config b/apps/emqx_authz/rebar.config index f3d365c47..354b04808 100644 --- a/apps/emqx_authz/rebar.config +++ b/apps/emqx_authz/rebar.config @@ -6,7 +6,8 @@ {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}}, {emqx_mongodb, {path, "../emqx_mongodb"}}, - {emqx_redis, {path, "../emqx_redis"}} + {emqx_redis, {path, "../emqx_redis"}}, + {emqx_mysql, {path, "../emqx_mysql"}} ]}. {shell, [ diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 50ea284fe..e9654557e 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -11,7 +11,8 @@ emqx_resource, emqx_connector, emqx_mongodb, - emqx_redis + emqx_redis, + emqx_mysql ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 01267a7b2..6b4722ade 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -70,7 +70,7 @@ fields(mongo_sharded) -> fields(mysql) -> authz_common_fields(mysql) ++ [{query, query()}] ++ - proplists:delete(prepare_statement, emqx_connector_mysql:fields(config)); + proplists:delete(prepare_statement, emqx_mysql:fields(config)); fields(postgresql) -> authz_common_fields(postgresql) ++ [{query, query()}] ++ diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index 768479e98..fb6a29c3d 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -54,13 +54,13 @@ create(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_mysql, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}. update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - case emqx_authz_utils:update_resource(emqx_connector_mysql, Source) of + case emqx_authz_utils:update_resource(emqx_mysql, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index e1e85d282..77be82ec2 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -389,7 +389,7 @@ cmd() -> connector_fields(DB) -> connector_fields(DB, config). -connector_fields(redis = DB, Fields) -> +connector_fields(DB, Fields) when DB =:= redis; DB =:= mysql -> connector_fields(DB, Fields, emqx); connector_fields(DB, Fields) -> connector_fields(DB, Fields, emqx_connector). diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index d276a2e1b..06449b3b4 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -44,7 +44,7 @@ init_per_suite(Config) -> {ok, _} = emqx_resource:create_local( ?MYSQL_RESOURCE, ?RESOURCE_GROUP, - emqx_connector_mysql, + emqx_mysql, mysql_config(), #{} ), diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md index 6baba29de..8cd6d4df9 100644 --- a/apps/emqx_connector/README.md +++ b/apps/emqx_connector/README.md @@ -3,25 +3,4 @@ This application is a collection of `connectors`. A `connector` is a callback module of `emqx_resource` that maintains the data related to -external resources. Put all resource related callback modules in a single application is good as -we can put some util functions/modules here for reusing purpose. - -For example, a MySQL connector is an emqx resource that maintains all the MySQL connection -related parameters (configs) and the TCP connections to the MySQL server. - -An MySQL connector can be used as following: - -``` -(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). -[#{config => - #{cacertfile => [],certfile => [], - database => "mqtt",keyfile => [],password => "public", - pool_size => 1, - server => {{127,0,0,1},3306}, - ssl => false,user => "root",verify => false}, - id => <<"mysql-abc">>,mod => emqx_connector_mysql, - state => #{poolname => 'mysql-abc'}, - status => connected}] -(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). -{ok,[<<"count(1)">>],[[1]]} -``` +external resources. diff --git a/apps/emqx_connector/docker-ct b/apps/emqx_connector/docker-ct index 000346e82..7ecf66a95 100644 --- a/apps/emqx_connector/docker-ct +++ b/apps/emqx_connector/docker-ct @@ -1,2 +1 @@ -mysql pgsql diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 59588093c..132863127 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -10,7 +10,6 @@ {emqx_utils, {path, "../emqx_utils"}}, {emqx_resource, {path, "../emqx_resource"}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}} ]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index eb82a0eec..d268a244a 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -13,8 +13,6 @@ eredis, epgsql, eldap2, - mysql, - mongodb, ehttpc, jose, emqx, diff --git a/apps/emqx_mysql/README.md b/apps/emqx_mysql/README.md new file mode 100644 index 000000000..04db049ed --- /dev/null +++ b/apps/emqx_mysql/README.md @@ -0,0 +1,15 @@ +# MySQL Connector + +This application houses the MySQL Database connector. +It provides the APIs to connect to MySQL Databases. + +It is used by the MySQL bridge to insert messages and by the emqx_authz and +emqx_authn applications to check user permissions. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [APL](../../APL.txt). diff --git a/apps/emqx_mysql/docker-ct b/apps/emqx_mysql/docker-ct new file mode 100644 index 000000000..0eaebf127 --- /dev/null +++ b/apps/emqx_mysql/docker-ct @@ -0,0 +1 @@ +mysql diff --git a/apps/emqx_mysql/rebar.config b/apps/emqx_mysql/rebar.config new file mode 100644 index 000000000..58b6665ad --- /dev/null +++ b/apps/emqx_mysql/rebar.config @@ -0,0 +1,9 @@ +%% -*- mode: erlang; -*- + +{erl_opts, [debug_info]}. +{deps, [ + %% NOTE: mind ecpool version when updating eredis_cluster version + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} +]}. diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src new file mode 100644 index 000000000..c0f8ec7e7 --- /dev/null +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -0,0 +1,14 @@ +{application, emqx_mysql, [ + {description, "EMQX MySQL Database Connector"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + mysql + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_mysql.erl rename to apps/emqx_mysql/src/emqx_mysql.erl index 2931a8ec5..2a4db3147 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -13,9 +13,9 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_mysql). +-module(emqx_mysql). --include("emqx_connector.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl similarity index 97% rename from apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl rename to apps/emqx_mysql/test/emqx_mysql_SUITE.erl index a0455c92c..b8886f1cd 100644 --- a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl +++ b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl @@ -13,18 +13,18 @@ % %% limitations under the License. % %%-------------------------------------------------------------------- --module(emqx_connector_mysql_SUITE). +-module(emqx_mysql_SUITE). -compile(nowarn_export_all). -compile(export_all). --include("emqx_connector.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). -define(MYSQL_HOST, "mysql"). --define(MYSQL_RESOURCE_MOD, emqx_connector_mysql). +-define(MYSQL_RESOURCE_MOD, emqx_mysql). all() -> emqx_common_test_helpers:all(?MODULE). @@ -60,7 +60,7 @@ end_per_testcase(_, _Config) -> t_lifecycle(_Config) -> perform_lifecycle_check( - <<"emqx_connector_mysql_SUITE">>, + <<"emqx_mysql_SUITE">>, mysql_config() ). diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 3b3be6ccf..ada341cfb 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -3,6 +3,7 @@ {deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + , {emqx_bridge, {path, "../../apps/emqx_mysql"}} , {emqx_utils, {path, "../emqx_utils"}} ]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 84d518804..f9e45a525 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -99,7 +99,7 @@ resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer; resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector; resource_type(mongodb_sharded) -> emqx_bridge_mongodb_connector; resource_type(mongodb_single) -> emqx_bridge_mongodb_connector; -resource_type(mysql) -> emqx_connector_mysql; +resource_type(mysql) -> emqx_mysql; resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector; resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector; resource_type(redis_single) -> emqx_bridge_redis_connector; diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index 4d041135f..01e253bd9 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -80,7 +80,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_connector_mysql:fields(config) -- + (emqx_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> [type_field(), name_field() | fields("config")]; diff --git a/mix.exs b/mix.exs index 67b58cff0..f7483aedb 100644 --- a/mix.exs +++ b/mix.exs @@ -378,6 +378,7 @@ defmodule EMQXUmbrella.MixProject do emqx_slow_subs: :permanent, emqx_mongodb: :permanent, emqx_redis: :permanent, + emqx_mysql: :permanent, emqx_plugins: :permanent, emqx_mix: :none ] ++ diff --git a/rebar.config.erl b/rebar.config.erl index 085972554..ca518323a 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -441,6 +441,7 @@ relx_apps(ReleaseType, Edition) -> emqx_slow_subs, emqx_mongodb, emqx_redis, + emqx_mysql, emqx_plugins ] ++ [quicer || is_quicer_supported()] ++ diff --git a/rel/i18n/emqx_connector_mysql.hocon b/rel/i18n/emqx_mysql.hocon similarity index 91% rename from rel/i18n/emqx_connector_mysql.hocon rename to rel/i18n/emqx_mysql.hocon index dd86b62c8..db6aa1b83 100644 --- a/rel/i18n/emqx_connector_mysql.hocon +++ b/rel/i18n/emqx_mysql.hocon @@ -1,4 +1,4 @@ -emqx_connector_mysql { +emqx_mysql { server.desc: """The IPv4 or IPv6 address or the hostname to connect to.