refactor: move MySQL open source connector to its own app

apps/emqx_authn/rebar.config

@@ -5,7 +5,8 @@
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_connector, {path, "../emqx_connector"}},
     {emqx_mongodb, {path, "../emqx_mongodb"}},
-    {emqx_redis, {path, "../emqx_redis"}}
+    {emqx_redis, {path, "../emqx_redis"}},
+    {emqx_mysql, {path, "../emqx_mysql"}}
 ]}.
 
 {edoc_opts, [{preprocess, true}]}.

apps/emqx_authn/src/emqx_authn.app.src

@@ -14,7 +14,8 @@
         mysql,
         jose,
         emqx_mongodb,
-        emqx_redis
+        emqx_redis,
+        emqx_mysql
     ]},
     {mod, {emqx_authn_app, []}},
     {env, []},

apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl

@@ -62,7 +62,7 @@ fields(mysql) ->
         {query, fun query/1},
         {query_timeout, fun query_timeout/1}
     ] ++ emqx_authn_schema:common_fields() ++
-        proplists:delete(prepare_statement, emqx_connector_mysql:fields(config)).
+        proplists:delete(prepare_statement, emqx_mysql:fields(config)).
 
 desc(mysql) ->
     ?DESC(mysql);
@@ -92,12 +92,12 @@ create(_AuthenticatorID, Config) ->
 create(Config0) ->
     ResourceId = emqx_authn_utils:make_resource_id(?MODULE),
     {Config, State} = parse_config(Config0),
-    {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_connector_mysql, Config),
+    {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config),
     {ok, State#{resource_id => ResourceId}}.
 
 update(Config0, #{resource_id := ResourceId} = _State) ->
     {Config, NState} = parse_config(Config0),
-    case emqx_authn_utils:update_resource(emqx_connector_mysql, Config, ResourceId) of
+    case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, _} ->

apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl

@@ -62,7 +62,7 @@ init_per_suite(Config) ->
             {ok, _} = emqx_resource:create_local(
                 ?MYSQL_RESOURCE,
                 ?RESOURCE_GROUP,
-                emqx_connector_mysql,
+                emqx_mysql,
                 mysql_config(),
                 #{}
             ),

apps/emqx_authz/rebar.config

@@ -6,7 +6,8 @@
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_connector, {path, "../emqx_connector"}},
     {emqx_mongodb, {path, "../emqx_mongodb"}},
-    {emqx_redis, {path, "../emqx_redis"}}
+    {emqx_redis, {path, "../emqx_redis"}},
+    {emqx_mysql, {path, "../emqx_mysql"}}
 ]}.
 
 {shell, [

apps/emqx_authz/src/emqx_authz.app.src

@@ -11,7 +11,8 @@
         emqx_resource,
         emqx_connector,
         emqx_mongodb,
-        emqx_redis
+        emqx_redis,
+        emqx_mysql
     ]},
     {env, []},
     {modules, []},

apps/emqx_authz/src/emqx_authz_api_schema.erl

@@ -70,7 +70,7 @@ fields(mongo_sharded) ->
 fields(mysql) ->
     authz_common_fields(mysql) ++
         [{query, query()}] ++
-        proplists:delete(prepare_statement, emqx_connector_mysql:fields(config));
+        proplists:delete(prepare_statement, emqx_mysql:fields(config));
 fields(postgresql) ->
     authz_common_fields(postgresql) ++
         [{query, query()}] ++

apps/emqx_authz/src/emqx_authz_mysql.erl

@@ -54,13 +54,13 @@ create(#{query := SQL} = Source0) ->
     {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS),
     ResourceId = emqx_authz_utils:make_resource_id(?MODULE),
     Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}},
-    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_mysql, Source),
+    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source),
     Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}.
 
 update(#{query := SQL} = Source0) ->
     {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS),
     Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}},
-    case emqx_authz_utils:update_resource(emqx_connector_mysql, Source) of
+    case emqx_authz_utils:update_resource(emqx_mysql, Source) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, Id} ->

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -389,7 +389,7 @@ cmd() ->
 
 connector_fields(DB) ->
     connector_fields(DB, config).
-connector_fields(redis = DB, Fields) ->
+connector_fields(DB, Fields) when DB =:= redis; DB =:= mysql ->
     connector_fields(DB, Fields, emqx);
 connector_fields(DB, Fields) ->
     connector_fields(DB, Fields, emqx_connector).

apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl

@@ -44,7 +44,7 @@ init_per_suite(Config) ->
             {ok, _} = emqx_resource:create_local(
                 ?MYSQL_RESOURCE,
                 ?RESOURCE_GROUP,
-                emqx_connector_mysql,
+                emqx_mysql,
                 mysql_config(),
                 #{}
             ),

apps/emqx_connector/README.md

@@ -3,25 +3,4 @@
 This application is a collection of `connectors`.
 
 A `connector` is a callback module of `emqx_resource` that maintains the data related to
-external resources. Put all resource related callback modules in a single application is good as
+external resources.
-we can put some util functions/modules here for reusing purpose.
-
-For example, a MySQL connector is an emqx resource that maintains all the MySQL connection
-related parameters (configs) and the TCP connections to the MySQL server.
-
-An MySQL connector can be used as following:
-
-```
-(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose().
-[#{config =>
-       #{cacertfile => [],certfile => [],
-         database => "mqtt",keyfile => [],password => "public",
-         pool_size => 1,
-         server => {{127,0,0,1},3306},
-         ssl => false,user => "root",verify => false},
-   id => <<"mysql-abc">>,mod => emqx_connector_mysql,
-   state => #{poolname => 'mysql-abc'},
-   status => connected}]
-(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}).
-{ok,[<<"count(1)">>],[[1]]}
-```

apps/emqx_connector/docker-ct

@@ -1,2 +1 @@
-mysql
 pgsql

apps/emqx_connector/rebar.config

@@ -10,7 +10,6 @@
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_resource, {path, "../emqx_resource"}},
     {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
-    {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
     {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}}
 ]}.
 

apps/emqx_connector/src/emqx_connector.app.src

@@ -13,8 +13,6 @@
         eredis,
         epgsql,
         eldap2,
-        mysql,
-        mongodb,
         ehttpc,
         jose,
         emqx,

apps/emqx_mysql/README.md

@@ -0,0 +1,15 @@
+# MySQL Connector
+
+This application houses the MySQL Database connector.
+It provides the APIs to connect to MySQL Databases.
+
+It is used by the MySQL bridge to insert messages and by the emqx_authz and
+emqx_authn applications to check user permissions.
+
+## Contributing
+
+Please see our [contributing.md](../../CONTRIBUTING.md).
+
+## License
+
+See [APL](../../APL.txt).

apps/emqx_mysql/docker-ct

@@ -0,0 +1 @@
+mysql

apps/emqx_mysql/rebar.config

@@ -0,0 +1,9 @@
+%% -*- mode: erlang; -*-
+
+{erl_opts, [debug_info]}.
+{deps, [
+        %% NOTE: mind ecpool version when updating eredis_cluster version
+        {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
+        {emqx_connector, {path, "../../apps/emqx_connector"}},
+        {emqx_resource, {path, "../../apps/emqx_resource"}}
+]}.

apps/emqx_mysql/src/emqx_mysql.app.src

@@ -0,0 +1,14 @@
+{application, emqx_mysql, [
+    {description, "EMQX MySQL Database Connector"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib,
+        mysql
+    ]},
+    {env, []},
+    {modules, []},
+
+    {links, []}
+]}.

apps/emqx_mysql/src/emqx_mysql.erl

@@ -13,9 +13,9 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_connector_mysql).
+-module(emqx_mysql).
 
--include("emqx_connector.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").

apps/emqx_mysql/test/emqx_mysql_SUITE.erl

@@ -13,18 +13,18 @@
 % %% limitations under the License.
 % %%--------------------------------------------------------------------
 
--module(emqx_connector_mysql_SUITE).
+-module(emqx_mysql_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include("emqx_connector.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("stdlib/include/assert.hrl").
 
 -define(MYSQL_HOST, "mysql").
--define(MYSQL_RESOURCE_MOD, emqx_connector_mysql).
+-define(MYSQL_RESOURCE_MOD, emqx_mysql).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -60,7 +60,7 @@ end_per_testcase(_, _Config) ->
 
 t_lifecycle(_Config) ->
     perform_lifecycle_check(
-        <<"emqx_connector_mysql_SUITE">>,
+        <<"emqx_mysql_SUITE">>,
         mysql_config()
     ).
 

lib-ee/emqx_ee_bridge/rebar.config

@@ -3,6 +3,7 @@
 {deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}
        , {emqx_resource, {path, "../../apps/emqx_resource"}}
        , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
+       , {emqx_bridge, {path, "../../apps/emqx_mysql"}}
        , {emqx_utils, {path, "../emqx_utils"}}
        ]}.
 

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -99,7 +99,7 @@ resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer;
 resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector;
 resource_type(mongodb_sharded) -> emqx_bridge_mongodb_connector;
 resource_type(mongodb_single) -> emqx_bridge_mongodb_connector;
-resource_type(mysql) -> emqx_connector_mysql;
+resource_type(mysql) -> emqx_mysql;
 resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector;
 resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector;
 resource_type(redis_single) -> emqx_bridge_redis_connector;

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl

@@ -80,7 +80,7 @@ fields("config") ->
                 #{desc => ?DESC("local_topic"), default => undefined}
             )}
     ] ++ emqx_resource_schema:fields("resource_opts") ++
-        (emqx_connector_mysql:fields(config) --
+        (emqx_mysql:fields(config) --
             emqx_connector_schema_lib:prepare_statement_fields());
 fields("post") ->
     [type_field(), name_field() | fields("config")];

mix.exs

@@ -378,6 +378,7 @@ defmodule EMQXUmbrella.MixProject do
         emqx_slow_subs: :permanent,
         emqx_mongodb: :permanent,
         emqx_redis: :permanent,
+        emqx_mysql: :permanent,
         emqx_plugins: :permanent,
         emqx_mix: :none
       ] ++

rebar.config.erl

@@ -441,6 +441,7 @@ relx_apps(ReleaseType, Edition) ->
             emqx_slow_subs,
             emqx_mongodb,
             emqx_redis,
+            emqx_mysql,
             emqx_plugins
         ] ++
         [quicer || is_quicer_supported()] ++

rel/i18n/emqx_mysql.hocon

@@ -1,4 +1,4 @@
-emqx_connector_mysql {
+emqx_mysql {
 
 server.desc:
 """The IPv4 or IPv6 address or the hostname to connect to.<br/>